From 17366f9c137c9bf24c8ef6e9424a7c36c4db8c39 Mon Sep 17 00:00:00 2001 From: canton-machine <48923836+canton-machine@users.noreply.github.com> Date: Mon, 16 May 2022 11:11:28 +0200 Subject: [PATCH] Update 2022-05-16.07 (#26) Reference commit: 34e687665 Co-authored-by: Canton --- .../commands/TopologyAdminCommands.scala | 3 +- .../client/commands/VaultAdminCommands.scala | 30 --- .../canton/console/AdminCommandRunner.scala | 5 +- .../canton/console/ConsoleEnvironment.scala | 13 +- .../canton/console/InstanceReference.scala | 4 +- .../commands/DomainAdministration.scala | 5 +- .../commands/LedgerApiAdministration.scala | 3 +- .../commands/ParticipantAdministration.scala | 10 +- .../commands/TopologyAdministration.scala | 12 +- .../commands/VaultAdministration.scala | 13 -- .../participant-init.canton | 4 +- .../examples/06-messaging/contact/daml.yaml | 2 +- .../examples/06-messaging/message/daml.yaml | 2 +- .../ConcurrentEnvironmentLimiter.scala | 41 +++- .../canton/integration/EnvironmentSetup.scala | 27 +-- .../src/main/daml/CantonExamples/daml.yaml | 2 +- .../crypto/admin/v0/vault_service.proto | 15 -- .../canton/h2/stable/V2__changes_for_2.3.sql | 5 +- .../postgres/stable/V2__changes_for_2.3.sql | 5 +- .../src/main/resources/rewrite-appender.xml | 9 + .../canton/crypto/CryptoApi.scala | 13 +- .../canton/crypto/CryptoFactory.scala | 13 +- .../crypto/{HkdfOps.scala => Hkdf.scala} | 103 +++++++--- .../com/digitalasset/canton/crypto/Hmac.scala | 80 +------- .../digitalasset/canton/crypto/Nonce.scala | 2 +- .../canton/crypto/ProtocolCryptoApi.scala | 23 +++ .../digitalasset/canton/crypto/Random.scala | 33 ++-- .../com/digitalasset/canton/crypto/Salt.scala | 67 ++++--- .../canton/crypto/SyncCryptoApiProvider.scala | 21 +- .../crypto/admin/grpc/GrpcVaultService.scala | 10 - .../crypto/provider/jce/JcePureCrypto.scala | 102 ++++++++-- .../crypto/provider/tink/TinkPureCrypto.scala | 53 ++++- .../crypto/store/CryptoPrivateStore.scala | 43 ----- .../crypto/store/CryptoPublicStore.scala | 10 + .../store/db/DbCryptoPrivateStore.scala | 64 +----- .../memory/InMemoryCryptoPrivateStore.scala | 20 +- .../canton/data/GenTransactionTree.scala | 7 +- .../environment/CantonNodeBootstrap.scala | 92 +++++++-- .../canton/protocol/DomainParameters.scala | 1 - .../protocol/messages/AcsCommitment.scala | 55 ++++-- .../messages/EncryptedViewMessage.scala | 12 +- .../protocol/messages/InformeeMessage.scala | 2 + .../MalformedMediatorRequestResult.scala | 43 +++-- .../protocol/messages/MediatorRequest.scala | 2 + .../protocol/messages/MediatorResponse.scala | 51 +++-- .../protocol/messages/MediatorResult.scala | 4 +- .../SignedProtocolMessageContent.scala | 6 +- .../messages/TranferOutMediatorMessage.scala | 9 +- .../messages/TransactionResultMessage.scala | 43 +++-- .../messages/TransferInMediatorMessage.scala | 9 +- .../protocol/messages/TransferResult.scala | 59 ++++-- .../authentication/AuthenticationToken.scala | 6 +- .../sequencing/client/SequencerClient.scala | 125 ++++++------ .../GrpcSequencerClientTransport.scala | 9 +- .../HasCryptographicEvidence.scala | 4 +- .../canton/topology/LegalIdentityInit.scala | 101 +++++++--- .../digitalasset/canton/topology/Member.scala | 6 +- .../canton/topology/TopologyManager.scala | 29 ++- .../client/CachingDomainTopologyClient.scala | 14 +- .../StoreBasedDomainTopologyClient.scala | 18 +- .../TopologyTimestampPlusEpsilonTracker.scala | 182 +++++++++--------- .../TopologyTransactionProcessor.scala | 132 ++++++++----- .../canton/topology/store/TopologyStore.scala | 51 +++-- .../store/TopologyTransactionCollection.scala | 4 +- .../topology/store/db/DbTopologyStore.scala | 64 +++--- .../store/memory/InMemoryTopologyStore.scala | 26 +-- .../SignedTopologyTransaction.scala | 6 +- .../version/HasProtocolVersionedWrapper.scala | 96 ++++++--- .../canton/crypto/EncryptionTest.scala | 10 +- .../digitalasset/canton/crypto/HkdfTest.scala | 166 +++++++++++----- .../digitalasset/canton/crypto/HmacTest.scala | 46 +---- .../canton/crypto/RandomTest.scala | 27 +++ .../digitalasset/canton/crypto/SaltTest.scala | 20 +- .../digitalasset/canton/crypto/TestHkdf.scala | 18 -- .../canton/crypto/TestHmacSecret.scala | 18 -- .../digitalasset/canton/crypto/TestSalt.scala | 6 +- .../crypto/provider/jce/JceCryptoTest.scala | 4 +- .../provider/symbolic/SymbolicCrypto.scala | 28 +-- .../symbolic/SymbolicCryptoTest.scala | 6 +- .../symbolic/SymbolicPureCrypto.scala | 44 ++++- .../crypto/provider/tink/TinkCryptoTest.scala | 4 +- .../crypto/store/CryptoPrivateStoreTest.scala | 24 --- .../store/db/DbCryptoPrivateStoreTest.scala | 3 +- .../canton/data/GenTransactionTreeTest.scala | 13 +- .../canton/data/MerkleTreeTest.scala | 4 +- .../canton/data/TransactionViewTest.scala | 4 +- .../canton/protocol/ExampleTransaction.scala | 4 +- .../protocol/ExampleTransactionFactory.scala | 17 +- .../protocol/messages/AcsCommitmentTest.scala | 3 + .../messages/MediatorResponseTest.scala | 3 + .../grpc/AuthenticationTokenManagerTest.scala | 6 +- .../SequencerClientAuthenticationTest.scala | 8 +- .../protocol/SequencedEventTest.scala | 1 + .../store/SequencedEventStoreTest.scala | 2 +- .../store/db/DatabaseDeadlockTest.scala | 2 +- .../topology/TestingIdentityFactory.scala | 14 +- ...ologyTimestampPlusEpsilonTrackerTest.scala | 143 ++++++++++---- .../topology/store/TopologyStoreTest.scala | 75 +++++++- .../canton/util/retry/PolicyTest.scala | 8 +- .../HasProtocolVersionedWrapperTest.scala | 2 +- .../demo/src/main/daml/ai-analysis/daml.yaml | 2 +- community/demo/src/main/daml/bank/daml.yaml | 2 +- community/demo/src/main/daml/doctor/daml.yaml | 2 +- .../src/main/daml/health-insurance/daml.yaml | 2 +- .../src/main/daml/medical-records/daml.yaml | 2 +- .../canton/domain/DomainNodeBootstrap.scala | 53 +++-- ...opologyManagerIdentityInitialization.scala | 61 +++--- .../EmbeddedMediatorInitialization.scala | 13 +- .../TopologyManagementInitialization.scala | 1 - .../ConfirmationResponseProcessor.scala | 23 ++- .../mediator/DomainNodeMediatorFactory.scala | 61 ------ .../canton/domain/mediator/Mediator.scala | 3 + .../mediator/MediatorRuntimeFactory.scala | 4 + .../admin/SequencerInitialization.scala | 30 ++- .../MemberAuthenticationService.scala | 4 +- .../topology/DomainTopologyManager.scala | 2 +- .../client/DomainInitializationObserver.scala | 2 +- .../ConfirmationResponseProcessorTest.scala | 4 + .../domain/mediator/MediatorStateTest.scala | 2 +- .../mediator/ResponseAggregationTest.scala | 17 +- .../store/FinalizedResponseStoreTest.scala | 2 +- .../AuthenticationTokenCacheTest.scala | 10 +- .../MemberAuthenticationServiceTest.scala | 4 +- .../MemberAuthenticationStoreTest.scala | 14 +- ...rAuthenticationServerInterceptorTest.scala | 7 +- .../GrpcSequencerIntegrationTest.scala | 4 +- .../service/GrpcSequencerServiceTest.scala | 7 +- ...ainTopologyManagerRequestServiceTest.scala | 2 +- community/participant/src/main/daml/daml.yaml | 2 +- .../canton/participant/ParticipantNode.scala | 37 ++-- .../participant/admin/RepairService.scala | 4 +- .../config/LocalParticipantConfig.scala | 12 +- .../domain/DomainRegistryHelpers.scala | 2 - .../grpc/ParticipantInitializeTopology.scala | 26 +-- .../api/CantonAdminTokenAuthService.scala | 9 +- .../api/CantonLedgerApiServerWrapper.scala | 5 +- .../BadRootHashMessagesRequestProcessor.scala | 3 + .../protocol/TransactionProcessingSteps.scala | 20 +- .../protocol/TransactionProcessor.scala | 7 +- .../ConfirmationRequestFactory.scala | 12 +- .../EncryptedViewMessageFactory.scala | 17 +- .../protocol/submission/SeedGenerator.scala | 121 +----------- .../submission/TransactionTreeFactory.scala | 4 +- .../TransactionTreeFactoryImpl.scala | 6 +- .../transfer/TransferInProcessingSteps.scala | 18 +- .../transfer/TransferOutProcessingSteps.scala | 19 +- .../transfer/TransferOutRequest.scala | 4 +- .../ConfirmationResponseFactory.scala | 4 + .../pruning/AcsCommitmentProcessor.scala | 11 +- .../memory/InMemoryMultiDomainEventLog.scala | 10 +- .../canton/participant/sync/SyncDomain.scala | 13 +- .../ParticipantTopologyDispatcher.scala | 2 +- .../topology/ParticipantTopologyManager.scala | 2 +- .../protocol/MessageDispatcherTest.scala | 13 +- .../ConfirmationRequestFactoryTest.scala | 32 ++- .../submission/SeedGeneratorTest.scala | 39 +--- .../TransferInProcessingStepsTest.scala | 157 +++++++-------- .../TransferOutProcessingStepsTest.scala | 32 ++- .../pruning/AcsCommitmentProcessorTest.scala | 19 +- .../store/AcsCommitmentStoreTest.scala | 48 ++++- .../participant/store/TransferStoreTest.scala | 17 +- .../ParticipantTopologyDispatcherTest.scala | 6 +- project/BuildCommon.scala | 20 +- project/Dependencies.scala | 5 +- project/project/DamlVersions.scala | 6 +- 165 files changed, 2172 insertions(+), 1800 deletions(-) rename community/common/src/main/scala/com/digitalasset/canton/crypto/{HkdfOps.scala => Hkdf.scala} (56%) create mode 100644 community/common/src/main/scala/com/digitalasset/canton/crypto/ProtocolCryptoApi.scala create mode 100644 community/common/src/test/scala/com/digitalasset/canton/crypto/RandomTest.scala delete mode 100644 community/common/src/test/scala/com/digitalasset/canton/crypto/TestHkdf.scala delete mode 100644 community/common/src/test/scala/com/digitalasset/canton/crypto/TestHmacSecret.scala delete mode 100644 community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/DomainNodeMediatorFactory.scala diff --git a/community/app/src/main/scala/com/digitalasset/canton/admin/api/client/commands/TopologyAdminCommands.scala b/community/app/src/main/scala/com/digitalasset/canton/admin/api/client/commands/TopologyAdminCommands.scala index 4bfab0dc6..1984d9c91 100644 --- a/community/app/src/main/scala/com/digitalasset/canton/admin/api/client/commands/TopologyAdminCommands.scala +++ b/community/app/src/main/scala/com/digitalasset/canton/admin/api/client/commands/TopologyAdminCommands.scala @@ -360,11 +360,12 @@ object TopologyAdminCommands { signedBy: Option[Fingerprint], domainId: DomainId, newParameters: DynamicDomainParameters, + force: Boolean, ) extends BaseCommand[v0.DomainParametersChangeAuthorization] { override def createRequest(): Either[String, v0.DomainParametersChangeAuthorization] = v0.DomainParametersChangeAuthorization( authorization = - authData(TopologyChangeOp.Replace, signedBy, replaceExisting = false, force = false), + authData(TopologyChangeOp.Replace, signedBy, replaceExisting = false, force = force), domain = domainId.toProtoPrimitive, parameters = Option(newParameters.toProtoV0), ).asRight diff --git a/community/app/src/main/scala/com/digitalasset/canton/admin/api/client/commands/VaultAdminCommands.scala b/community/app/src/main/scala/com/digitalasset/canton/admin/api/client/commands/VaultAdminCommands.scala index c14c537d1..4e2d5994a 100644 --- a/community/app/src/main/scala/com/digitalasset/canton/admin/api/client/commands/VaultAdminCommands.scala +++ b/community/app/src/main/scala/com/digitalasset/canton/admin/api/client/commands/VaultAdminCommands.scala @@ -249,34 +249,4 @@ object VaultAdminCommands { ) } - case class RotateHmacSecret(length: Int) - extends BaseVaultAdminCommand[ - v0.RotateHmacSecretRequest, - v0.RotateHmacSecretResponse, - Unit, - ] { - - override def createRequest(): Either[String, v0.RotateHmacSecretRequest] = - Right( - v0.RotateHmacSecretRequest( - length = length - ) - ) - - override def submitRequest( - service: VaultServiceStub, - request: v0.RotateHmacSecretRequest, - ): Future[v0.RotateHmacSecretResponse] = { - service.rotateHmacSecret(request) - } - - override def handleResponse( - response: v0.RotateHmacSecretResponse - ): Either[String, Unit] = Right(()) - - // may time some time if we need to wait for entropy - override def timeoutType: TimeoutType = DefaultUnboundedTimeout - - } - } diff --git a/community/app/src/main/scala/com/digitalasset/canton/console/AdminCommandRunner.scala b/community/app/src/main/scala/com/digitalasset/canton/console/AdminCommandRunner.scala index 5587eac80..c0287300e 100644 --- a/community/app/src/main/scala/com/digitalasset/canton/console/AdminCommandRunner.scala +++ b/community/app/src/main/scala/com/digitalasset/canton/console/AdminCommandRunner.scala @@ -10,7 +10,6 @@ import com.digitalasset.canton.console.CommandErrors.ConsoleTimeout import com.digitalasset.canton.crypto.Crypto import com.digitalasset.canton.environment.{CantonNode, CantonNodeBootstrap} import com.digitalasset.canton.logging.{NamedLogging, TracedLogger} -import com.digitalasset.canton.tracing.NoTracing import scala.annotation.tailrec @@ -103,7 +102,7 @@ trait BaseInspection[I <: CantonNode] { } -trait FeatureFlagFilter extends NamedLogging with NoTracing { +trait FeatureFlagFilter extends NamedLogging { protected def consoleEnvironment: ConsoleEnvironment @@ -113,7 +112,7 @@ trait FeatureFlagFilter extends NamedLogging with NoTracing { if (flag) { command } else { - logger.error( + noTracingLogger.error( s"The command is currently disabled. You need to enable it explicitly by setting `canton.features.${config} = yes` in your Canton configuration file (`.conf`)" ) throw new CommandFailure() diff --git a/community/app/src/main/scala/com/digitalasset/canton/console/ConsoleEnvironment.scala b/community/app/src/main/scala/com/digitalasset/canton/console/ConsoleEnvironment.scala index 1364203b8..fbd6ba54c 100644 --- a/community/app/src/main/scala/com/digitalasset/canton/console/ConsoleEnvironment.scala +++ b/community/app/src/main/scala/com/digitalasset/canton/console/ConsoleEnvironment.scala @@ -7,7 +7,7 @@ import ammonite.util.Bind import com.digitalasset.canton.DomainAlias import com.digitalasset.canton.admin.api.client.data.CantonStatus import com.digitalasset.canton.config.RequireTypes.{InstanceName, NonNegativeInt} -import com.digitalasset.canton.config.{ConsoleCommandTimeout, TimeoutDuration} +import com.digitalasset.canton.config.{ConsoleCommandTimeout, ProcessingTimeout, TimeoutDuration} import com.digitalasset.canton.console.CommandErrors.{ CantonCommandError, CommandInternalError, @@ -17,6 +17,7 @@ import com.digitalasset.canton.console.Help.{Description, Summary, Topic} import com.digitalasset.canton.crypto.Fingerprint import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.environment.Environment +import com.digitalasset.canton.lifecycle.{FlagCloseable, Lifecycle} import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging} import com.digitalasset.canton.sequencing.{GrpcSequencerConnection, SequencerConnection} import com.digitalasset.canton.time.{NonNegativeFiniteDuration, SimClock} @@ -38,7 +39,7 @@ case class NodeReferences[A, R <: A, L <: A](local: Seq[L], remote: Seq[R]) { /** The environment in which console commands are evaluated. */ @SuppressWarnings(Array("org.wartremover.warts.Any")) // required for `Binding[_]` usage -trait ConsoleEnvironment extends NamedLogging with AutoCloseable with NoTracing { +trait ConsoleEnvironment extends NamedLogging with FlagCloseable with NoTracing { type Env <: Environment type DomainLocalRef <: LocalDomainReference type DomainRemoteRef <: RemoteDomainReference @@ -129,6 +130,8 @@ trait ConsoleEnvironment extends NamedLogging with AutoCloseable with NoTracing } + protected def timeouts: ProcessingTimeout = environment.config.parameters.timeouts.processing + /** @return maximum runtime of a console command */ def commandTimeouts: ConsoleCommandTimeout = commandTimeoutReference.get() @@ -441,10 +444,8 @@ trait ConsoleEnvironment extends NamedLogging with AutoCloseable with NoTracing */ protected def selfAlias(): Bind[_] = Bind(ConsoleEnvironmentBinding.BindingName, this) - override def close(): Unit = { - grpcAdminCommandRunner.close() - environment.close() - tracerProvider.close() + override def onClosed(): Unit = { + Lifecycle.close(grpcAdminCommandRunner, environment, tracerProvider)(logger) } def startAll(): Unit = { diff --git a/community/app/src/main/scala/com/digitalasset/canton/console/InstanceReference.scala b/community/app/src/main/scala/com/digitalasset/canton/console/InstanceReference.scala index b6d239680..de9dbee4a 100644 --- a/community/app/src/main/scala/com/digitalasset/canton/console/InstanceReference.scala +++ b/community/app/src/main/scala/com/digitalasset/canton/console/InstanceReference.scala @@ -25,7 +25,7 @@ import com.digitalasset.canton.participant.config.{ import com.digitalasset.canton.protocol.{LfContractId, SerializableContractWithWitnesses} import com.digitalasset.canton.sequencing.SequencerConnection import com.digitalasset.canton.topology.{DomainId, Identity, ParticipantId} -import com.digitalasset.canton.tracing.TraceContext +import com.digitalasset.canton.tracing.{NoTracing, TraceContext} import com.digitalasset.canton.util.ErrorUtil import scala.util.hashing.MurmurHash3 @@ -82,7 +82,7 @@ trait InstanceReference /** Pointer for a potentially running instance by instance type (domain/participant) and its id. * These methods define the REPL interface for these instances (e.g. participant1 start) */ -trait LocalInstanceReference extends InstanceReference { +trait LocalInstanceReference extends InstanceReference with NoTracing { val name: String val consoleEnvironment: ConsoleEnvironment diff --git a/community/app/src/main/scala/com/digitalasset/canton/console/commands/DomainAdministration.scala b/community/app/src/main/scala/com/digitalasset/canton/console/commands/DomainAdministration.scala index 20a4e885e..c200c344d 100644 --- a/community/app/src/main/scala/com/digitalasset/canton/console/commands/DomainAdministration.scala +++ b/community/app/src/main/scala/com/digitalasset/canton/console/commands/DomainAdministration.scala @@ -19,6 +19,7 @@ import com.digitalasset.canton.topology._ import com.digitalasset.canton.topology.admin.grpc.BaseQuery import com.digitalasset.canton.topology.store.{TimeQuery, TopologyStoreId} import com.digitalasset.canton.topology.transaction._ +import com.google.protobuf.ByteString trait DomainAdministration { this: AdminCommandRunner => @@ -117,8 +118,8 @@ trait DomainAdministration { ) @Help.Summary("Set the Dynamic Domain Parameters configured for the domain") - def set_dynamic_domain_parameters(dynamicDomainParameters: DynamicDomainParameters) = - topology.domain_parameters_changes.authorize(id, dynamicDomainParameters).discard + def set_dynamic_domain_parameters(dynamicDomainParameters: DynamicDomainParameters): Unit = + topology.domain_parameters_changes.authorize(id, dynamicDomainParameters).discard[ByteString] @Help.Summary("Update the Dynamic Domain Parameters for the domain") def update_dynamic_parameters( diff --git a/community/app/src/main/scala/com/digitalasset/canton/console/commands/LedgerApiAdministration.scala b/community/app/src/main/scala/com/digitalasset/canton/console/commands/LedgerApiAdministration.scala index c77cb441d..ae3b9eb28 100644 --- a/community/app/src/main/scala/com/digitalasset/canton/console/commands/LedgerApiAdministration.scala +++ b/community/app/src/main/scala/com/digitalasset/canton/console/commands/LedgerApiAdministration.scala @@ -54,6 +54,7 @@ import com.digitalasset.canton.metrics.MetricHandle import com.digitalasset.canton.networking.grpc.{GrpcError, RecordingStreamObserver} import com.digitalasset.canton.protocol.LfContractId import com.digitalasset.canton.topology.{DomainId, ParticipantId, PartyId} +import com.digitalasset.canton.tracing.NoTracing import com.digitalasset.canton.util.ResourceUtil import com.digitalasset.canton.{LedgerTransactionId, LfPartyId} import io.grpc.StatusRuntimeException @@ -64,7 +65,7 @@ import java.util.concurrent.TimeoutException import java.util.concurrent.atomic.AtomicReference import scala.concurrent.Await -trait BaseLedgerApiAdministration { +trait BaseLedgerApiAdministration extends NoTracing { this: LedgerApiCommandRunner with NamedLogging with FeatureFlagFilter => implicit protected val consoleEnvironment: ConsoleEnvironment diff --git a/community/app/src/main/scala/com/digitalasset/canton/console/commands/ParticipantAdministration.scala b/community/app/src/main/scala/com/digitalasset/canton/console/commands/ParticipantAdministration.scala index 2ba9e34cc..336fc05dc 100644 --- a/community/app/src/main/scala/com/digitalasset/canton/console/commands/ParticipantAdministration.scala +++ b/community/app/src/main/scala/com/digitalasset/canton/console/commands/ParticipantAdministration.scala @@ -63,6 +63,7 @@ import com.digitalasset.canton.sequencing.{ import com.digitalasset.canton.serialization.ProtoConverter import com.digitalasset.canton.time.{DomainTimeTrackerConfig, NonNegativeFiniteDuration} import com.digitalasset.canton.topology.{DomainId, ParticipantId, PartyId} +import com.digitalasset.canton.tracing.NoTracing import com.digitalasset.canton.util.ShowUtil._ import com.digitalasset.canton.util._ @@ -306,7 +307,8 @@ class LocalParticipantTestingGroup( consoleEnvironment: ConsoleEnvironment, loggerFactory: NamedLoggerFactory, ) extends ParticipantTestingGroup(participantRef, consoleEnvironment, loggerFactory) - with FeatureFlagFilter { + with FeatureFlagFilter + with NoTracing { import participantRef._ @Help.Summary("Lookup contracts in the Private Contract Store", FeatureFlag.Testing) @@ -557,7 +559,8 @@ class LocalParticipantPruningAdministrationGroup( runner: AdminCommandRunner with LedgerApiCommandRunner with BaseInspection[ParticipantNode], consoleEnvironment: ConsoleEnvironment, loggerFactory: NamedLoggerFactory, -) extends ParticipantPruningAdministrationGroup(runner, consoleEnvironment, loggerFactory) { +) extends ParticipantPruningAdministrationGroup(runner, consoleEnvironment, loggerFactory) + with NoTracing { import runner._ @@ -579,7 +582,8 @@ class LocalCommitmentsAdministrationGroup( val consoleEnvironment: ConsoleEnvironment, val loggerFactory: NamedLoggerFactory, ) extends FeatureFlagFilter - with Helpful { + with Helpful + with NoTracing { import runner._ diff --git a/community/app/src/main/scala/com/digitalasset/canton/console/commands/TopologyAdministration.scala b/community/app/src/main/scala/com/digitalasset/canton/console/commands/TopologyAdministration.scala index 106eae2ab..22722abdf 100644 --- a/community/app/src/main/scala/com/digitalasset/canton/console/commands/TopologyAdministration.scala +++ b/community/app/src/main/scala/com/digitalasset/canton/console/commands/TopologyAdministration.scala @@ -1177,11 +1177,11 @@ class TopologyAdministrationGroup( object domain_parameters_changes extends Helpful { @Help.Summary("Change domain parameters") @Help.Description("""Authorize a transaction to change parameters of the domain. - domainId: Id of the domain affected by the change. - newParameters: New value of the domain parameters. - signedBy: Refers to the fingerprint of the authorizing key which in turn must be authorized by a valid, locally existing certificate. - If none is given, a key is automatically determined. - synchronize: Synchronize timeout can be used to ensure that the state has been propagated into the node + |domainId: Id of the domain affected by the change. + |newParameters: New value of the domain parameters. + |signedBy: Refers to the fingerprint of the authorizing key which in turn must be authorized by a valid, locally existing certificate. + | If none is given, a key is automatically determined. + |synchronize: Synchronize timeout can be used to ensure that the state has been propagated into the node """) def authorize( domainId: DomainId, @@ -1194,7 +1194,7 @@ class TopologyAdministrationGroup( consoleEnvironment.run { adminCommand( TopologyAdminCommands.Write - .AuthorizeDomainParametersChange(signedBy, domainId, newParameters) + .AuthorizeDomainParametersChange(signedBy, domainId, newParameters, force = false) ) } ) diff --git a/community/app/src/main/scala/com/digitalasset/canton/console/commands/VaultAdministration.scala b/community/app/src/main/scala/com/digitalasset/canton/console/commands/VaultAdministration.scala index ec40cc097..1f1129bd4 100644 --- a/community/app/src/main/scala/com/digitalasset/canton/console/commands/VaultAdministration.scala +++ b/community/app/src/main/scala/com/digitalasset/canton/console/commands/VaultAdministration.scala @@ -74,19 +74,6 @@ class SecretKeyAdministration(runner: AdminCommandRunner, consoleEnvironment: Co } } - @Help.Summary("Rotate the HMAC secret") - @Help.Description( - """Replace the stored HMAC secret with a new generated secret of the given length. - - length: Length of the HMAC secret. Must be at least 128 bits, but less than the internal block size of the hash function. - |""" - ) - def rotate_hmac_secret(length: Int = HmacSecret.defaultLength): Unit = { - consoleEnvironment.run { - adminCommand(VaultAdminCommands.RotateHmacSecret(length)) - } - } - } class LocalSecretKeyAdministration( diff --git a/community/app/src/pack/examples/03-advanced-configuration/participant-init.canton b/community/app/src/pack/examples/03-advanced-configuration/participant-init.canton index 1f5093076..86c47327e 100644 --- a/community/app/src/pack/examples/03-advanced-configuration/participant-init.canton +++ b/community/app/src/pack/examples/03-advanced-configuration/participant-init.canton @@ -9,10 +9,12 @@ if(participant.domains.list_registered().isEmpty) { // above connect operation is asynchronous. it is generally at the discretion of the domain // to decide if a participant can join and when. therefore, we need to asynchronously wait here - // until the participant observes his activation on the domain + // until the participant observes its activation on the domain utils.retry_until_true { participant.domains.active("mydomain") } + // synchronize vetting to ensure the participant has the package needed for the ping + participant.packages.synchronize_vetting() // verify that the connection works participant.health.ping(participant) diff --git a/community/app/src/pack/examples/06-messaging/contact/daml.yaml b/community/app/src/pack/examples/06-messaging/contact/daml.yaml index 45753cb09..47b920cf0 100644 --- a/community/app/src/pack/examples/06-messaging/contact/daml.yaml +++ b/community/app/src/pack/examples/06-messaging/contact/daml.yaml @@ -1,4 +1,4 @@ -sdk-version: 2.2.0-snapshot.20220504.9851.0.4c8e027d +sdk-version: 2.3.0-snapshot.20220511.9887.0.5a509164 sandbox-options: - --wall-clock-time name: contact diff --git a/community/app/src/pack/examples/06-messaging/message/daml.yaml b/community/app/src/pack/examples/06-messaging/message/daml.yaml index c9dfd2a60..7dad7d9ed 100644 --- a/community/app/src/pack/examples/06-messaging/message/daml.yaml +++ b/community/app/src/pack/examples/06-messaging/message/daml.yaml @@ -1,4 +1,4 @@ -sdk-version: 2.2.0-snapshot.20220504.9851.0.4c8e027d +sdk-version: 2.3.0-snapshot.20220511.9887.0.5a509164 sandbox-options: - --wall-clock-time name: message diff --git a/community/app/src/test/scala/com/digitalasset/canton/integration/ConcurrentEnvironmentLimiter.scala b/community/app/src/test/scala/com/digitalasset/canton/integration/ConcurrentEnvironmentLimiter.scala index 2535010e6..df0ae9b86 100644 --- a/community/app/src/test/scala/com/digitalasset/canton/integration/ConcurrentEnvironmentLimiter.scala +++ b/community/app/src/test/scala/com/digitalasset/canton/integration/ConcurrentEnvironmentLimiter.scala @@ -3,8 +3,10 @@ package com.digitalasset.canton.integration -import java.util.concurrent.Semaphore +import com.typesafe.scalalogging.LazyLogging +import java.util.concurrent.Semaphore +import java.util.concurrent.atomic.AtomicReference import scala.util.control.NonFatal /** Although our integration tests are designed not to conflict with one another when running concurrently, @@ -22,7 +24,15 @@ import scala.util.control.NonFatal * This may well activate a slow poke notification in ScalaTest, however these are left in place to support discovery * of any tests that fail to halt entirely or run for an abnormally long amount of time. */ -object ConcurrentEnvironmentLimiter { +object ConcurrentEnvironmentLimiter extends LazyLogging { + + private sealed trait State + private object New extends State + private object Queued extends State + private object Running extends State + private object Failed extends State + private object Done extends State + val IntegrationTestConcurrencyLimit = "canton-test.integration.concurrency" private val concurrencyLimit: Int = System.getProperty(IntegrationTestConcurrencyLimit, "1").toInt @@ -30,23 +40,44 @@ object ConcurrentEnvironmentLimiter { /** Configured to be fair so earlier started tests will be first to get environments */ private val semaphore = new Semaphore(concurrencyLimit, true) + /** contains a map from test name to state (queue, run) */ + private val active = new AtomicReference[Map[String, State]](Map.empty) + + private def change(name: String, state: State, purge: Boolean = false): Unit = { + val current = active + .getAndUpdate { cur => + if (purge) cur.removed(name) + else cur.updated(name, state) + } + val currentState = current.getOrElse(name, New) + val currentSet = current.keys.toSet + val numSet = if (purge) (currentSet - name) else (currentSet + name) + logger.debug(s"${name}: $currentState => $state (${numSet.size} pending)") + } + /** Block an environment creation until a permit is available. */ - def create[A](block: => A): A = { + def create[A](name: String)(block: => A): A = { + change(name, Queued) scala.concurrent.blocking { semaphore.acquire() } + change(name, Running) try block catch { // creations can easily fail and throw // capture these and immediately release the permit as the destroy method will not be called case NonFatal(e) => semaphore.release() + change(name, Failed, purge = true) throw e } } /** Attempt to destroy an environment and ensure that the permit is released */ - def destroy[A](block: => A): A = + def destroy[A](name: String)(block: => A): A = try block - finally semaphore.release() + finally { + semaphore.release() + change(name, Done, purge = true) + } } diff --git a/community/app/src/test/scala/com/digitalasset/canton/integration/EnvironmentSetup.scala b/community/app/src/test/scala/com/digitalasset/canton/integration/EnvironmentSetup.scala index a652b8541..d9f164848 100644 --- a/community/app/src/test/scala/com/digitalasset/canton/integration/EnvironmentSetup.scala +++ b/community/app/src/test/scala/com/digitalasset/canton/integration/EnvironmentSetup.scala @@ -3,6 +3,7 @@ package com.digitalasset.canton.integration +import com.digitalasset.canton.CloseableTest import com.digitalasset.canton.environment.Environment import com.digitalasset.canton.logging.NamedLogging import org.scalatest.{BeforeAndAfterAll, Suite} @@ -97,25 +98,27 @@ sealed trait EnvironmentSetup[E <: Environment, TCE <: TestConsoleEnvironment[E] } protected def createEnvironment(): TCE = - ConcurrentEnvironmentLimiter.create(manualCreateEnvironment()) - - protected def destroyEnvironment(environment: TCE): Unit = ConcurrentEnvironmentLimiter.destroy { - val config = environment.actualConfig - plugins.foreach(_.beforeEnvironmentDestroyed(config, environment)) - try { - environment.close() - } finally { - envDef.teardown(()) - plugins.foreach(_.afterEnvironmentDestroyed(config)) + ConcurrentEnvironmentLimiter.create(getClass.getName)(manualCreateEnvironment()) + + protected def destroyEnvironment(environment: TCE): Unit = + ConcurrentEnvironmentLimiter.destroy(getClass.getName) { + val config = environment.actualConfig + plugins.foreach(_.beforeEnvironmentDestroyed(config, environment)) + try { + environment.close() + } finally { + envDef.teardown(()) + plugins.foreach(_.afterEnvironmentDestroyed(config)) + } } - } } /** Starts an environment in a beforeAll test and uses it for all tests. * Destroys it in an afterAll hook. */ trait SharedEnvironment[E <: Environment, TCE <: TestConsoleEnvironment[E]] - extends EnvironmentSetup[E, TCE] { + extends EnvironmentSetup[E, TCE] + with CloseableTest { this: Suite with HasEnvironmentDefinition[E, TCE] with NamedLogging => @SuppressWarnings(Array("org.wartremover.warts.Var")) diff --git a/community/common/src/main/daml/CantonExamples/daml.yaml b/community/common/src/main/daml/CantonExamples/daml.yaml index a56d3fe54..9a78e3e8f 100644 --- a/community/common/src/main/daml/CantonExamples/daml.yaml +++ b/community/common/src/main/daml/CantonExamples/daml.yaml @@ -1,4 +1,4 @@ -sdk-version: 2.2.0-snapshot.20220504.9851.0.4c8e027d +sdk-version: 2.3.0-snapshot.20220511.9887.0.5a509164 name: CantonExamples exposed-modules: - CantonExamples diff --git a/community/common/src/main/protobuf/com/digitalasset/canton/crypto/admin/v0/vault_service.proto b/community/common/src/main/protobuf/com/digitalasset/canton/crypto/admin/v0/vault_service.proto index 95b138ed7..f2e4f337e 100644 --- a/community/common/src/main/protobuf/com/digitalasset/canton/crypto/admin/v0/vault_service.proto +++ b/community/common/src/main/protobuf/com/digitalasset/canton/crypto/admin/v0/vault_service.proto @@ -81,11 +81,6 @@ service VaultService { */ rpc ListCertificates(ListCertificateRequest) returns (ListCertificateResponse); - /** - * Rotate the stored HMAC secret. - */ - rpc RotateHmacSecret(RotateHmacSecretRequest) returns (RotateHmacSecretResponse); - } message GenerateCertificateRequest { @@ -171,13 +166,3 @@ message GenerateEncryptionKeyRequest { message GenerateEncryptionKeyResponse { com.digitalasset.canton.crypto.v0.EncryptionPublicKey public_key = 1; } - -message RotateHmacSecretRequest { - - // Length of the HMAC secret. Must be at least 128 bits, but less than the internal block size of the hash function. - int32 length = 1; -} - -message RotateHmacSecretResponse { - -} diff --git a/community/common/src/main/resources/db/migration/canton/h2/stable/V2__changes_for_2.3.sql b/community/common/src/main/resources/db/migration/canton/h2/stable/V2__changes_for_2.3.sql index 55f020cb7..4b60ffc12 100644 --- a/community/common/src/main/resources/db/migration/canton/h2/stable/V2__changes_for_2.3.sql +++ b/community/common/src/main/resources/db/migration/canton/h2/stable/V2__changes_for_2.3.sql @@ -2,4 +2,7 @@ -- SPDX-License-Identifier: Apache-2.0 -- TODO(#9014) migrate to non null -ALTER TABLE topology_transactions ADD sequenced bigint null; \ No newline at end of file +ALTER TABLE topology_transactions ADD sequenced bigint null; + +-- A stored HMAC secret is not used anymore +DROP TABLE crypto_hmac_secret; diff --git a/community/common/src/main/resources/db/migration/canton/postgres/stable/V2__changes_for_2.3.sql b/community/common/src/main/resources/db/migration/canton/postgres/stable/V2__changes_for_2.3.sql index 55f020cb7..4b60ffc12 100644 --- a/community/common/src/main/resources/db/migration/canton/postgres/stable/V2__changes_for_2.3.sql +++ b/community/common/src/main/resources/db/migration/canton/postgres/stable/V2__changes_for_2.3.sql @@ -2,4 +2,7 @@ -- SPDX-License-Identifier: Apache-2.0 -- TODO(#9014) migrate to non null -ALTER TABLE topology_transactions ADD sequenced bigint null; \ No newline at end of file +ALTER TABLE topology_transactions ADD sequenced bigint null; + +-- A stored HMAC secret is not used anymore +DROP TABLE crypto_hmac_secret; diff --git a/community/common/src/main/resources/rewrite-appender.xml b/community/common/src/main/resources/rewrite-appender.xml index b17c87e98..927081d13 100644 --- a/community/common/src/main/resources/rewrite-appender.xml +++ b/community/common/src/main/resources/rewrite-appender.xml @@ -255,6 +255,15 @@ true + + + com.daml.platform.apiserver.services.transaction.ApiTransactionService + INFO + Unhandled internal error + java.util.concurrent.RejectedExecutionException + true + + diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/CryptoApi.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/CryptoApi.scala index 3da9cd010..8f3d71e97 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/CryptoApi.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/CryptoApi.scala @@ -68,13 +68,18 @@ class Crypto( override def onClosed(): Unit = Lifecycle.close(cryptoPrivateStore, cryptoPublicStore)(logger) } -trait CryptoPureApi extends EncryptionOps with SigningOps with HmacOps with HkdfOps with HashOps -trait CryptoPrivateApi extends EncryptionPrivateOps with SigningPrivateOps with HmacPrivateOps +trait CryptoPureApi + extends EncryptionOps + with SigningOps + with HmacOps + with HkdfOps + with HashOps + with RandomOps +trait CryptoPrivateApi extends EncryptionPrivateOps with SigningPrivateOps trait CryptoPrivateStoreApi extends CryptoPrivateApi with EncryptionPrivateStoreOps with SigningPrivateStoreOps - with HmacPrivateStoreOps sealed trait SyncCryptoError extends Product with Serializable with PrettyPrinting object SyncCryptoError { @@ -84,11 +89,13 @@ object SyncCryptoError { owner: KeyOwner, keyPurpose: KeyPurpose, timestamp: CantonTimestamp, + candidates: Seq[Fingerprint], ) extends SyncCryptoError { override def pretty: Pretty[KeyNotAvailable] = prettyOfClass( param("owner", _.owner), param("key purpose", _.keyPurpose), param("timestamp", _.timestamp), + param("candidates", _.candidates), ) } diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/CryptoFactory.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/CryptoFactory.scala index 8f86c6f99..60f3f2cb0 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/CryptoFactory.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/CryptoFactory.scala @@ -27,7 +27,6 @@ import com.digitalasset.canton.crypto.provider.tink.{ import com.digitalasset.canton.crypto.store.{CryptoPrivateStore, CryptoPublicStore} import com.digitalasset.canton.logging.NamedLoggerFactory import com.digitalasset.canton.resource.Storage -import com.digitalasset.canton.tracing.TraceContext import org.bouncycastle.jce.provider.BouncyCastleProvider import java.security.Security @@ -84,8 +83,7 @@ object CryptoFactory { timeouts: ProcessingTimeout, loggerFactory: NamedLoggerFactory, )(implicit - ec: ExecutionContext, - traceContext: TraceContext, + ec: ExecutionContext ): EitherT[Future, String, Crypto] = { val cryptoPrivateStore = CryptoPrivateStore.create(storage, timeouts, loggerFactory) val cryptoPublicStore = CryptoPublicStore.create(storage, timeouts, loggerFactory) @@ -132,7 +130,7 @@ object CryptoFactory { encryptionKeyScheme, cryptoPrivateStore, ) - EitherT.rightT( + EitherT.rightT[Future, String]( new Crypto( pureCrypto, privateCrypto, @@ -144,13 +142,6 @@ object CryptoFactory { ) ) } - // Initialize the HMAC secret for an active replica - _ <- - if (storage.isActive) - crypto.privateCrypto - .initializeHmacSecret() - .leftMap(err => s"Failed to initialize HMAC secret: $err") - else EitherT.rightT[Future, String](()) } yield crypto } diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/HkdfOps.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/Hkdf.scala similarity index 56% rename from community/common/src/main/scala/com/digitalasset/canton/crypto/HkdfOps.scala rename to community/common/src/main/scala/com/digitalasset/canton/crypto/Hkdf.scala index 2f7323556..45497b6fa 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/HkdfOps.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/Hkdf.scala @@ -3,8 +3,7 @@ package com.digitalasset.canton.crypto -import cats.syntax.either._ -import cats.syntax.foldable._ +import com.digitalasset.canton.crypto.HkdfError.HkdfKeyTooShort import com.digitalasset.canton.data.ViewPosition.MerklePathElement import com.digitalasset.canton.logging.pretty.{Pretty, PrettyPrinting} import com.google.common.annotations.VisibleForTesting @@ -16,9 +15,42 @@ import com.google.protobuf.ByteString trait HkdfOps { this: HmacOps => - import HkdfError._ + /** Sanity check the parameters to the HKDF before calling the internal implementations. */ + private def checkParameters( + outputBytes: Int, + algorithm: HmacAlgorithm = defaultHmacAlgorithm, + ): Either[HkdfError, Unit] = { + import HkdfError._ - /** Produce a new secret from the given secret and purpose + for { + _ <- Either.cond(outputBytes >= 0, (), HkdfOutputNegative(outputBytes)) + hashBytes = algorithm.hashAlgorithm.length + nrChunks = scala.math.ceil(outputBytes.toDouble / hashBytes).toInt + _ <- Either + .cond[HkdfError, Unit]( + nrChunks <= 255, + (), + HkdfOutputTooLong(length = outputBytes, maximum = hashBytes * 255), + ) + } yield () + } + + protected def hkdfExpandInternal( + keyMaterial: SecureRandomness, + outputBytes: Int, + info: HkdfInfo, + algorithm: HmacAlgorithm = defaultHmacAlgorithm, + ): Either[HkdfError, SecureRandomness] + + protected def computeHkdfInternal( + keyMaterial: ByteString, + outputBytes: Int, + info: HkdfInfo, + salt: ByteString = ByteString.EMPTY, + algorithm: HmacAlgorithm = defaultHmacAlgorithm, + ): Either[HkdfError, SecureRandomness] + + /** Produce a new secret from the given secret and purpose. Only performs the expand step of the HKDF from RFC 5869. * * @param keyMaterial Cryptographically secure, uniformly random initial key material. Must be at least as long as * the length of the hash function chosen for the HMAC scheme. @@ -28,39 +60,44 @@ trait HkdfOps { * @param info Specify the purpose of the derived key (optional). Note that you can derive multiple * independent keys from the same key material by varying the purpose. * @param algorithm The hash algorithm to be used for the HKDF construction - * @return */ def hkdfExpand( keyMaterial: SecureRandomness, outputBytes: Int, info: HkdfInfo, algorithm: HmacAlgorithm = defaultHmacAlgorithm, - ): Either[HkdfError, SecureRandomness] = { - for { - _ <- Either.cond(outputBytes >= 0, (), HkdfOutputNegative(outputBytes)) - hashBytes = algorithm.hashAlgorithm.length - _ <- Either.cond[HkdfError, Unit]( - keyMaterial.unwrap.size >= hashBytes, - (), - HkdfKeyTooShort(length = keyMaterial.unwrap.size, needed = hashBytes), - ) - prk <- HmacSecret.create(keyMaterial.unwrap).leftMap(HkdfHmacError) - nrChunks = scala.math.ceil(outputBytes.toDouble / hashBytes).toInt - _ <- Either - .cond[HkdfError, Unit]( - nrChunks <= 255, - (), - HkdfOutputTooLong(length = outputBytes, maximum = hashBytes * 255), - ) - outputAndLast <- (1 to nrChunks).toList - .foldM(ByteString.EMPTY -> ByteString.EMPTY) { case ((out, last), chunk) => - val chunkByte = ByteString.copyFrom(Array[Byte](chunk.toByte)) - hmacWithSecret(prk, last.concat(info.bytes).concat(chunkByte), algorithm) - .bimap(HkdfHmacError, hmac => out.concat(hmac.unwrap) -> hmac.unwrap) - } - (out, _last) = outputAndLast - } yield SecureRandomness(out.substring(0, outputBytes)) - } + ): Either[HkdfError, SecureRandomness] = for { + _ <- checkParameters(outputBytes, algorithm) + hashBytes = algorithm.hashAlgorithm.length + _ <- Either.cond[HkdfError, Unit]( + keyMaterial.unwrap.size >= hashBytes, + (), + HkdfKeyTooShort(length = keyMaterial.unwrap.size, needed = hashBytes), + ) + expansion <- hkdfExpandInternal(keyMaterial, outputBytes, info, algorithm) + } yield expansion + + /** Produce a new secret from the given key material using the HKDF from RFC 5869 with both extract and expand phases. + * + * @param keyMaterial Input key material from which to derive another key. + * @param outputBytes The length of the produced secret. May be at most 255 times the size of the output of the + * selected hashing algorithm. If you need to derive multiple keys, set the `info` parameter + * to different values, for each key that you need. + * @param info Specify the purpose of the derived key (optional). Note that you can derive multiple + * independent keys from the same key material by varying the purpose. + * @param salt Optional salt. Should be set if the input key material is not cryptographically secure, uniformly random. + * @param algorithm The hash algorithm to be used for the HKDF construction + */ + def computeHkdf( + keyMaterial: ByteString, + outputBytes: Int, + info: HkdfInfo, + salt: ByteString = ByteString.EMPTY, + algorithm: HmacAlgorithm = defaultHmacAlgorithm, + ): Either[HkdfError, SecureRandomness] = for { + _ <- checkParameters(outputBytes, algorithm) + expansion <- computeHkdfInternal(keyMaterial, outputBytes, info, salt, algorithm) + } yield expansion } @@ -114,4 +151,8 @@ object HkdfError { ) } + case class HkdfInternalError(error: String) extends HkdfError { + override def pretty: Pretty[HkdfInternalError] = prettyOfClass(unnamedParam(_.error.unquoted)) + } + } diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/Hmac.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/Hmac.scala index c0f3753dd..e7e8f01bc 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/Hmac.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/Hmac.scala @@ -3,25 +3,21 @@ package com.digitalasset.canton.crypto -import java.security.{InvalidKeyException, NoSuchAlgorithmException} -import cats.data.EitherT import cats.syntax.either._ import com.digitalasset.canton.ProtoDeserializationError -import com.digitalasset.canton.crypto.store.{CryptoPrivateStore, CryptoPrivateStoreError} +import com.digitalasset.canton.crypto.store.CryptoPrivateStoreError import com.digitalasset.canton.logging.pretty.{Pretty, PrettyInstances, PrettyPrinting, PrettyUtil} import com.digitalasset.canton.serialization.DeserializationError import com.digitalasset.canton.serialization.ProtoConverter.ParsingResult import com.digitalasset.canton.store.db.DbSerializationException -import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.util.NoCopy import com.digitalasset.canton.version.HasProtoV0 import com.google.protobuf.ByteString +import slick.jdbc.{GetResult, SetParameter} +import java.security.{InvalidKeyException, NoSuchAlgorithmException} import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec -import slick.jdbc.{GetResult, SetParameter} - -import scala.concurrent.{ExecutionContext, Future} sealed abstract class HmacAlgorithm(val name: String, val hashAlgorithm: HashAlgorithm) extends PrettyPrinting { @@ -35,7 +31,7 @@ object HmacAlgorithm { val algorithms: Seq[HmacAlgorithm] = Seq(HmacSha256) - case object HmacSha256 extends HmacAlgorithm("HmacSHA256", HashAlgorithm.Sha256) { + case object HmacSha256 extends HmacAlgorithm("HMACSHA256", HashAlgorithm.Sha256) { override def toProtoEnum: v0.HmacAlgorithm = v0.HmacAlgorithm.HmacSha256 } @@ -172,9 +168,9 @@ object HmacSecret { * NOTE: The length of the HMAC secret should not exceed the internal _block_ size of the hash function, * e.g., 512 bits for SHA256. */ - def generate(length: Int = defaultLength): HmacSecret = { + def generate(randomOps: RandomOps, length: Int = defaultLength): HmacSecret = { require(length >= defaultLength, s"Specified HMAC secret key length ${length} too small.") - new HmacSecret(SecureRandomness.randomByteString(length)) + new HmacSecret(randomOps.generateRandomByteString(length)) } } @@ -192,70 +188,6 @@ trait HmacOps { } -/** HMAC operations that require access to private key store and knowledge about the current entity to key association */ -trait HmacPrivateOps extends HmacOps { - - protected def hmacSecret(implicit - ec: ExecutionContext, - traceContext: TraceContext, - ): EitherT[Future, HmacError, Option[HmacSecret]] - - /** Calculates the HMAC of the given message using the stored private key */ - def hmac(message: ByteString, algorithm: HmacAlgorithm = defaultHmacAlgorithm)(implicit - executionContext: ExecutionContext - ): EitherT[Future, HmacError, Hmac] = - hmacSecret(executionContext, TraceContext.empty) - .subflatMap(_.toRight(HmacError.MissingHmacSecret)) - .subflatMap { secret => - hmacWithSecret(secret, message, algorithm) - } - - /** Initializes the private HMAC secret if not present */ - def initializeHmacSecret(length: Int = HmacSecret.defaultLength)(implicit - executionContext: ExecutionContext, - traceContext: TraceContext, - ): EitherT[Future, HmacError, Unit] - - /** Rotates the private HMAC secret by replacing the existing one with a newly generated secret. */ - def rotateHmacSecret(length: Int = HmacSecret.defaultLength)(implicit - executionContext: ExecutionContext, - traceContext: TraceContext, - ): EitherT[Future, HmacError, Unit] - -} - -/** A default implementation with a private key store for the HMAC secret. */ -trait HmacPrivateStoreOps extends HmacPrivateOps { - - protected val store: CryptoPrivateStore - - override def hmacSecret(implicit - ec: ExecutionContext, - traceContext: TraceContext, - ): EitherT[Future, HmacError, Option[HmacSecret]] = - store.hmacSecret - .leftMap[HmacError](HmacError.HmacPrivateStoreError) - - override def initializeHmacSecret(length: Int = HmacSecret.defaultLength)(implicit - ec: ExecutionContext, - traceContext: TraceContext, - ): EitherT[Future, HmacError, Unit] = - for { - secretOption <- hmacSecret - _ <- secretOption.fold[EitherT[Future, HmacError, Unit]]( - store.storeHmacSecret(HmacSecret.generate()).leftMap(HmacError.HmacPrivateStoreError) - )(_ => EitherT.rightT(())) - } yield () - - override def rotateHmacSecret(length: Int = HmacSecret.defaultLength)(implicit - ec: ExecutionContext, - traceContext: TraceContext, - ): EitherT[Future, HmacError, Unit] = { - store.storeHmacSecret(HmacSecret.generate()).leftMap(HmacError.HmacPrivateStoreError) - } - -} - sealed trait HmacError extends Product with Serializable with PrettyPrinting object HmacError { diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/Nonce.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/Nonce.scala index 9e4f2964b..183d7e830 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/Nonce.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/Nonce.scala @@ -48,7 +48,7 @@ object Nonce { .getOrElse(throw new DbSerializationException(s"Could not deserialize nonce from db")) } - def generate(): Nonce = new Nonce(SecureRandomness.randomByteString(length)) + def generate(randomOps: RandomOps): Nonce = new Nonce(randomOps.generateRandomByteString(length)) def fromProtoPrimitive(bytes: ByteString): ParsingResult[Nonce] = Either.cond( diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/ProtocolCryptoApi.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/ProtocolCryptoApi.scala new file mode 100644 index 000000000..5562e80af --- /dev/null +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/ProtocolCryptoApi.scala @@ -0,0 +1,23 @@ +// Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.digitalasset.canton.crypto + +import com.digitalasset.canton.version.ProtocolVersion + +/** Helper methods to select the appropriate crypto primitive for a particular protocol version. */ +object ProtocolCryptoApi { + + def hkdf(hkdfOps: HkdfOps, protocolVersion: ProtocolVersion)( + keyMaterial: SecureRandomness, + outputBytes: Int, + info: HkdfInfo, + ): Either[HkdfError, SecureRandomness] = + protocolVersion match { + case ProtocolVersion.unstable_development => + hkdfOps.computeHkdf(keyMaterial.unwrap, outputBytes, info) + case _ => + hkdfOps.hkdfExpand(keyMaterial, outputBytes, info) + } + +} diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/Random.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/Random.scala index 9b911a55d..efc5b4c39 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/Random.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/Random.scala @@ -4,18 +4,27 @@ package com.digitalasset.canton.crypto import com.digitalasset.canton.serialization.{DeserializationError, HasCryptographicEvidence} -import com.digitalasset.canton.util.NoCopy import com.google.protobuf.ByteString import scala.util.Random +trait RandomOps { + + protected def generateRandomBytes(length: Int): Array[Byte] + + def generateRandomByteString(length: Int): ByteString = + ByteString.copyFrom(generateRandomBytes(length)) + + def generateSecureRandomness(length: Int): SecureRandomness = SecureRandomness( + generateRandomByteString(length) + ) +} + /** The class is a tag that denotes a byte string as a securely generated random value. * * Not an AnyVal as we also want it to be a serializable value such that we can encrypt it. */ -case class SecureRandomness private[crypto] (unwrap: ByteString) - extends HasCryptographicEvidence - with NoCopy { +sealed abstract case class SecureRandomness(unwrap: ByteString) extends HasCryptographicEvidence { override def getCryptographicEvidence: ByteString = unwrap } @@ -23,19 +32,7 @@ case class SecureRandomness private[crypto] (unwrap: ByteString) object SecureRandomness { private[crypto] def apply(unwrap: ByteString): SecureRandomness = - new SecureRandomness(unwrap) - - private val rand = new java.security.SecureRandom() - - def randomBytes(length: Int): Array[Byte] = { - val secretBytes = new Array[Byte](length) - rand.nextBytes(secretBytes) - secretBytes - } - - def randomByteString(length: Int): ByteString = ByteString.copyFrom(randomBytes(length)) - - def secureRandomness(length: Int): SecureRandomness = SecureRandomness(randomByteString(length)) + new SecureRandomness(unwrap) {} /** Recover secure randomness from a byte string. Use for deserialization only. Fails if the provided byte string * is not of the expected length. @@ -50,7 +47,7 @@ object SecureRandomness { bytes, ) ) - else Right(new SecureRandomness(bytes)) + else Right(SecureRandomness(bytes)) } } diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/Salt.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/Salt.scala index 2854c7db3..462a49d17 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/Salt.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/Salt.scala @@ -3,7 +3,6 @@ package com.digitalasset.canton.crypto -import cats.data.EitherT import cats.syntax.either._ import com.digitalasset.canton.ProtoDeserializationError import com.digitalasset.canton.logging.pretty.{Pretty, PrettyPrinting} @@ -13,7 +12,24 @@ import com.digitalasset.canton.util.NoCopy import com.digitalasset.canton.version.HasProtoV0 import com.google.protobuf.ByteString -import scala.concurrent.{ExecutionContext, Future} +/** A seed to derive further salts from. + * + * Unlike [[Salt]] this seed will not be shipped to another participant. + */ +abstract sealed case class SaltSeed(unwrap: ByteString) + +object SaltSeed { + + /** Default length for a salt seed is 128 bits */ + val defaultLength = 16 + + private[crypto] def apply(bytes: ByteString): SaltSeed = { + new SaltSeed(bytes) {} + } + + def generate(length: Int = defaultLength)(randomOps: RandomOps): SaltSeed = + SaltSeed(randomOps.generateRandomByteString(length)) +} /** Indicates the algorithm used to generate and derive salts. */ sealed trait SaltAlgorithm extends Product with Serializable with PrettyPrinting { @@ -79,20 +95,14 @@ object Salt { SaltError.InvalidSaltCreation(bytes, algorithm), ) - /** Derives a salt from a `seed` salt and an `index`. */ - def deriveSalt(seed: Salt, index: Int, hmacOps: HmacOps): Either[SaltError, Salt] = { - deriveSalt(seed, DeterministicEncoding.encodeInt(index), hmacOps) - } - - def tryDeriveSalt(seed: Salt, index: Int, hmacOps: HmacOps): Salt = { - deriveSalt(seed, index, hmacOps).valueOr(err => throw new IllegalStateException(err.toString)) - } - - /** Derives a salt from a `seed` salt and `bytes` using an HMAC as a pseudo-random function. */ - def deriveSalt(seed: Salt, bytes: ByteString, hmacOps: HmacOps): Either[SaltError, Salt] = + private def deriveSalt( + seed: ByteString, + bytes: ByteString, + hmacOps: HmacOps, + ): Either[SaltError, Salt] = for { pseudoSecret <- HmacSecret - .create(seed.toByteString) + .create(seed) .leftMap(SaltError.HmacGenerationError) saltAlgorithm = SaltAlgorithm.Hmac(hmacOps.defaultHmacAlgorithm) hmac <- hmacOps @@ -101,20 +111,27 @@ object Salt { salt <- create(hmac.unwrap, saltAlgorithm) } yield salt - def tryDeriveSalt(seed: Salt, bytes: ByteString, hmacOps: HmacOps): Salt = - deriveSalt(seed, bytes, hmacOps).valueOr(err => throw new IllegalStateException(err.toString)) + /** Derives a salt from a `seed` salt and an `index`. */ + def deriveSalt(seed: SaltSeed, index: Int, hmacOps: HmacOps): Either[SaltError, Salt] = { + deriveSalt(seed, DeterministicEncoding.encodeInt(index), hmacOps) + } - /** Generates a salt from the given `bytes` using HMAC with a stored HMAC secret as pseudo-random function. */ - def generate(bytes: ByteString, hmacPrivateOps: HmacPrivateOps)(implicit - ec: ExecutionContext - ): EitherT[Future, SaltError, Salt] = { - val saltAlgorithm = SaltAlgorithm.Hmac(hmacPrivateOps.defaultHmacAlgorithm) - hmacPrivateOps - .hmac(bytes, saltAlgorithm.hmacAlgorithm) - .leftMap(SaltError.HmacGenerationError) - .flatMap(hmac => create(hmac.unwrap, saltAlgorithm).toEitherT) + def tryDeriveSalt(seed: SaltSeed, index: Int, hmacOps: HmacOps): Salt = { + deriveSalt(seed, index, hmacOps).valueOr(err => throw new IllegalStateException(err.toString)) } + /** Derives a salt from a `seed` salt and `bytes` using an HMAC as a pseudo-random function. */ + def deriveSalt(seed: SaltSeed, bytes: ByteString, hmacOps: HmacOps): Either[SaltError, Salt] = + deriveSalt(seed.unwrap, bytes, hmacOps) + + def tryDeriveSalt(seed: SaltSeed, bytes: ByteString, hmacOps: HmacOps): Salt = + deriveSalt(seed, bytes, hmacOps).valueOr(err => throw new IllegalStateException(err.toString)) + + def tryDeriveSalt(seed: Salt, bytes: ByteString, hmacOps: HmacOps): Salt = + deriveSalt(seed.toByteString, bytes, hmacOps).valueOr(err => + throw new IllegalStateException(err.toString) + ) + def fromProtoV0(saltP: v0.Salt): ParsingResult[Salt] = for { saltAlgorithm <- SaltAlgorithm.fromProtoOneOf("algorithm", saltP.algorithm) diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/SyncCryptoApiProvider.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/SyncCryptoApiProvider.scala index 3ea1fe2eb..451536df0 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/SyncCryptoApiProvider.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/SyncCryptoApiProvider.scala @@ -169,7 +169,13 @@ class DomainSyncCryptoClient( .leftMap[SyncCryptoError](SyncCryptoError.StoreError) kk <- existingKeys.lastOption .toRight[SyncCryptoError]( - SyncCryptoError.KeyNotAvailable(owner, KeyPurpose.Signing, snapshot.timestamp) + SyncCryptoError + .KeyNotAvailable( + owner, + KeyPurpose.Signing, + snapshot.timestamp, + signingKeys.map(_.fingerprint), + ) ) .toEitherT[Future] } yield kk.fingerprint @@ -316,10 +322,16 @@ class DomainSnapshotSyncCryptoApi( .map { keyO => keyO .toRight( - KeyNotAvailable(owner, KeyPurpose.Encryption, ipsSnapshot.timestamp): SyncCryptoError + KeyNotAvailable( + owner, + KeyPurpose.Encryption, + ipsSnapshot.timestamp, + Seq.empty, + ): SyncCryptoError ) } ) + // TODO (error handling) better alert / error message if given key does not exist locally .flatMap(key => crypto.privateCrypto .decrypt(encryptedMessage, key.fingerprint)(deserialize) @@ -342,8 +354,11 @@ class DomainSnapshotSyncCryptoApi( .encryptionKey(owner) .map { keyO => keyO - .toRight(KeyNotAvailable(owner, KeyPurpose.Encryption, ipsSnapshot.timestamp)) + .toRight( + KeyNotAvailable(owner, KeyPurpose.Encryption, ipsSnapshot.timestamp, Seq.empty) + ) .flatMap(k => + // TODO (error handling): better error message if given key does not exist locally crypto.pureCrypto .encryptWith(message, k, version) .leftMap(SyncCryptoEncryptionError) diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/admin/grpc/GrpcVaultService.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/admin/grpc/GrpcVaultService.scala index e1cac1417..7faf16577 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/admin/grpc/GrpcVaultService.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/admin/grpc/GrpcVaultService.scala @@ -6,7 +6,6 @@ package com.digitalasset.canton.crypto.admin.grpc import cats.syntax.either._ import cats.syntax.traverseFilter._ import com.digitalasset.canton.crypto.admin.v0 -import com.digitalasset.canton.crypto.admin.v0.{RotateHmacSecretRequest, RotateHmacSecretResponse} import com.digitalasset.canton.crypto.{v0 => cryptoproto, _} import com.digitalasset.canton.topology.UniqueIdentifier import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging} @@ -191,13 +190,4 @@ class GrpcVaultService( ) EitherTUtil.toFuture(res) } - - override def rotateHmacSecret( - request: RotateHmacSecretRequest - ): Future[RotateHmacSecretResponse] = - TraceContext.fromGrpcContext { implicit traceContext => - EitherTUtil - .toFuture(mapErr(crypto.privateCrypto.rotateHmacSecret(request.length))) - .map(_ => RotateHmacSecretResponse()) - } } diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/provider/jce/JcePureCrypto.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/provider/jce/JcePureCrypto.scala index ca2fd4651..6e6671d2c 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/provider/jce/JcePureCrypto.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/provider/jce/JcePureCrypto.scala @@ -3,17 +3,8 @@ package com.digitalasset.canton.crypto.provider.jce -import java.security.interfaces.{ECPrivateKey, ECPublicKey} -import java.security.{ - GeneralSecurityException, - InvalidKeyException, - NoSuchAlgorithmException, - SignatureException, - PrivateKey => JPrivateKey, - PublicKey => JPublicKey, - Signature => JSignature, -} import cats.syntax.either._ +import com.digitalasset.canton.crypto.HkdfError.HkdfInternalError import com.digitalasset.canton.crypto._ import com.digitalasset.canton.serialization.DeserializationError import com.digitalasset.canton.util.ShowUtil @@ -24,9 +15,43 @@ import com.google.crypto.tink.subtle._ import com.google.crypto.tink.{Aead, PublicKeySign, PublicKeyVerify} import com.google.protobuf.ByteString import org.bouncycastle.asn1.gm.GMObjectIdentifiers +import org.bouncycastle.crypto.digests.SHA256Digest +import org.bouncycastle.crypto.generators.HKDFBytesGenerator +import org.bouncycastle.crypto.params.HKDFParameters +import java.security.interfaces.{ECPrivateKey, ECPublicKey} +import java.security.{ + GeneralSecurityException, + InvalidKeyException, + NoSuchAlgorithmException, + SecureRandom, + SignatureException, + PrivateKey => JPrivateKey, + PublicKey => JPublicKey, + Signature => JSignature, +} import scala.collection.concurrent.TrieMap +object JceSecureRandom { + + /** Uses [[ThreadLocal]] here to reduce contention and improve performance. */ + private val random: ThreadLocal[SecureRandom] = new ThreadLocal[SecureRandom] { + override def initialValue(): SecureRandom = newSecureRandom() + } + + private def newSecureRandom() = { + val rand = new SecureRandom() + rand.nextLong() + rand + } + + private[jce] def generateRandomBytes(length: Int): Array[Byte] = { + val randBytes = new Array[Byte](length) + random.get().nextBytes(randBytes) + randBytes + } +} + class JcePureCrypto( javaKeyConverter: JceJavaConverter, override val defaultSymmetricKeyScheme: SymmetricKeyScheme, @@ -167,7 +192,7 @@ class JcePureCrypto( ): Either[EncryptionKeyGenerationError, SymmetricKey] = scheme match { case SymmetricKeyScheme.Aes128Gcm => - val key128 = ByteString.copyFrom(SecureRandomness.randomBytes(scheme.keySizeInBytes)) + val key128 = generateRandomByteString(scheme.keySizeInBytes) Right(SymmetricKey.create(CryptoKeyFormat.Raw, key128, scheme)) } @@ -473,4 +498,59 @@ class JcePureCrypto( ) } + private def hkdf( + params: HKDFParameters, + outputBytes: Int, + algorithm: HmacAlgorithm, + ): Either[HkdfError, SecureRandomness] = { + val output = Array.fill[Byte](outputBytes)(0) + val digest = algorithm match { + case HmacAlgorithm.HmacSha256 => new SHA256Digest() + } + val generator = new HKDFBytesGenerator(digest) + + for { + generated <- + Either + .catchNonFatal { + generator.init(params) + generator.generateBytes(output, 0, outputBytes) + } + .leftMap(err => HkdfInternalError(show"Failed to compute HKDF with JCE: $err")) + _ <- Either.cond( + generated == outputBytes, + (), + HkdfInternalError(s"Generated only $generated bytes instead of $outputBytes"), + ) + expansion <- SecureRandomness + .fromByteString(outputBytes)(ByteString.copyFrom(output)) + .leftMap(err => HkdfInternalError(s"Invalid output from HKDF: $err")) + } yield expansion + } + + override protected def computeHkdfInternal( + keyMaterial: ByteString, + outputBytes: Int, + info: HkdfInfo, + salt: ByteString, + algorithm: HmacAlgorithm, + ): Either[HkdfError, SecureRandomness] = { + val params = + new HKDFParameters(keyMaterial.toByteArray, salt.toByteArray, info.bytes.toByteArray) + hkdf(params, outputBytes, algorithm) + } + + override protected def hkdfExpandInternal( + keyMaterial: SecureRandomness, + outputBytes: Int, + info: HkdfInfo, + algorithm: HmacAlgorithm, + ): Either[HkdfError, SecureRandomness] = { + val params = + HKDFParameters.skipExtractParameters(keyMaterial.unwrap.toByteArray, info.bytes.toByteArray) + hkdf(params, outputBytes, algorithm) + } + + override protected def generateRandomBytes(length: Int): Array[Byte] = + JceSecureRandom.generateRandomBytes(length) } diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/provider/tink/TinkPureCrypto.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/provider/tink/TinkPureCrypto.scala index 23dc745dc..ce6db2f05 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/provider/tink/TinkPureCrypto.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/provider/tink/TinkPureCrypto.scala @@ -5,13 +5,16 @@ package com.digitalasset.canton.crypto.provider.tink import java.security.GeneralSecurityException import cats.syntax.either._ +import cats.syntax.foldable._ +import com.digitalasset.canton.util.ShowUtil._ +import com.digitalasset.canton.crypto.HkdfError.{HkdfHmacError, HkdfInternalError} import com.digitalasset.canton.crypto._ import com.digitalasset.canton.serialization.DeserializationError import com.digitalasset.canton.version.{HasVersionedToByteString, ProtocolVersion} import com.google.crypto.tink._ import com.google.crypto.tink.aead.AeadKeyTemplates import com.google.crypto.tink.config.TinkConfig -import com.google.crypto.tink.subtle.AesGcmJce +import com.google.crypto.tink.subtle.{AesGcmJce, Hkdf, Random} import com.google.protobuf.ByteString import scala.collection.concurrent.TrieMap @@ -240,6 +243,54 @@ class TinkPureCrypto private ( ) } yield () + override protected def computeHkdfInternal( + keyMaterial: ByteString, + outputBytes: Int, + info: HkdfInfo, + salt: ByteString, + algorithm: HmacAlgorithm, + ): Either[HkdfError, SecureRandomness] = { + Either + .catchOnly[GeneralSecurityException] { + Hkdf.computeHkdf( + algorithm.name, + keyMaterial.toByteArray, + salt.toByteArray, + info.bytes.toByteArray, + outputBytes, + ) + } + .leftMap(err => show"Failed to compute HKDF with Tink: $err") + .flatMap { hkdfOutput => + SecureRandomness + .fromByteString(outputBytes)(ByteString.copyFrom(hkdfOutput)) + .leftMap(err => s"Invalid output from HKDF: $err") + } + .leftMap(HkdfInternalError) + } + + override protected def hkdfExpandInternal( + keyMaterial: SecureRandomness, + outputBytes: Int, + info: HkdfInfo, + algorithm: HmacAlgorithm, + ): Either[HkdfError, SecureRandomness] = { + // NOTE: Tink does not expose only the expand phase, thus we have to implement it ourselves + val hashBytes = algorithm.hashAlgorithm.length + for { + prk <- HmacSecret.create(keyMaterial.unwrap).leftMap(HkdfHmacError) + nrChunks = scala.math.ceil(outputBytes.toDouble / hashBytes).toInt + outputAndLast <- (1 to nrChunks).toList + .foldM(ByteString.EMPTY -> ByteString.EMPTY) { case ((out, last), chunk) => + val chunkByte = ByteString.copyFrom(Array[Byte](chunk.toByte)) + hmacWithSecret(prk, last.concat(info.bytes).concat(chunkByte), algorithm) + .bimap(HkdfHmacError, hmac => out.concat(hmac.unwrap) -> hmac.unwrap) + } + (out, _last) = outputAndLast + } yield SecureRandomness(out.substring(0, outputBytes)) + } + + override protected def generateRandomBytes(length: Int): Array[Byte] = Random.randBytes(length) } object TinkPureCrypto { diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/store/CryptoPrivateStore.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/store/CryptoPrivateStore.scala index c75d847c8..64eb0c5ae 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/store/CryptoPrivateStore.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/store/CryptoPrivateStore.scala @@ -3,7 +3,6 @@ package com.digitalasset.canton.crypto.store -import java.util.concurrent.atomic.AtomicReference import cats.data.EitherT import cats.syntax.functor._ import com.digitalasset.canton.config.ProcessingTimeout @@ -71,7 +70,6 @@ trait CryptoPrivateStore extends AutoCloseable { this: NamedLogging => // Cached values for keys and secret protected val signingKeyMap: TrieMap[Fingerprint, SigningPrivateKeyWithName] = TrieMap.empty protected val decryptionKeyMap: TrieMap[Fingerprint, EncryptionPrivateKeyWithName] = TrieMap.empty - protected val hmacSecretRef: AtomicReference[Option[HmacSecret]] = new AtomicReference(None) // Write methods that the underlying store has to implement. protected def writeSigningKey(key: SigningPrivateKey, name: Option[KeyName])(implicit @@ -80,9 +78,6 @@ trait CryptoPrivateStore extends AutoCloseable { this: NamedLogging => protected def writeDecryptionKey(key: EncryptionPrivateKey, name: Option[KeyName])(implicit traceContext: TraceContext ): EitherT[Future, CryptoPrivateStoreError, Unit] - protected def writeHmacSecret(hmacSecret: HmacSecret)(implicit - traceContext: TraceContext - ): EitherT[Future, CryptoPrivateStoreError, Unit] @VisibleForTesting private[store] def listSigningKeys(implicit @@ -92,10 +87,6 @@ trait CryptoPrivateStore extends AutoCloseable { this: NamedLogging => private[store] def listDecryptionKeys(implicit traceContext: TraceContext ): EitherT[Future, CryptoPrivateStoreError, Set[EncryptionPrivateKeyWithName]] - @VisibleForTesting - private[store] def loadHmacSecret()(implicit - traceContext: TraceContext - ): EitherT[Future, CryptoPrivateStoreError, Option[HmacSecret]] protected def deletePrivateKey(keyId: Fingerprint)(implicit traceContext: TraceContext @@ -188,23 +179,6 @@ trait CryptoPrivateStore extends AutoCloseable { this: NamedLogging => val _ = decryptionKeyMap.put(key.id, EncryptionPrivateKeyWithName(key, name)) } - private[crypto] def hmacSecret(implicit - traceContext: TraceContext - ): EitherT[Future, CryptoPrivateStoreError, Option[HmacSecret]] = - hmacSecretRef.get() match { - case hmacSecret @ Some(_) => EitherT.rightT(hmacSecret) - case None => - loadHmacSecret().map { result => - hmacSecretRef.set(result) - result - } - } - - def storeHmacSecret(hmacSecret: HmacSecret)(implicit - traceContext: TraceContext - ): EitherT[Future, CryptoPrivateStoreError, Unit] = - writeHmacSecret(hmacSecret).map(_ => hmacSecretRef.set(Some(hmacSecret))) - private def retrieveAndUpdateCache[KN <: PrivateKeyWithName]( cache: TrieMap[Fingerprint, KN], readKey: Fingerprint => EitherT[Future, CryptoPrivateStoreError, Option[KN]], @@ -232,23 +206,6 @@ object CryptoPrivateStore { sealed trait CryptoPrivateStoreError extends Product with Serializable with PrettyPrinting object CryptoPrivateStoreError { - case class FailedToLoadHmacSecret(reason: String) extends CryptoPrivateStoreError { - override def pretty: Pretty[FailedToLoadHmacSecret] = prettyOfClass( - unnamedParam(_.reason.unquoted) - ) - } - - case object HmacSecretAlreadyExists extends CryptoPrivateStoreError { - override def pretty: Pretty[HmacSecretAlreadyExists.type] = - prettyOfObject[HmacSecretAlreadyExists.type] - } - - case class FailedToInsertHmacSecret(reason: String) extends CryptoPrivateStoreError { - override def pretty: Pretty[FailedToInsertHmacSecret] = prettyOfClass( - unnamedParam(_.reason.unquoted) - ) - } - case class FailedToListKeys(reason: String) extends CryptoPrivateStoreError { override def pretty: Pretty[FailedToListKeys] = prettyOfClass(unnamedParam(_.reason.unquoted)) } diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/store/CryptoPublicStore.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/store/CryptoPublicStore.scala index 3520e37b0..344be9d13 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/store/CryptoPublicStore.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/store/CryptoPublicStore.scala @@ -76,6 +76,16 @@ trait CryptoPublicStore extends AutoCloseable { case KeyPurpose.Encryption => encryptionKey(keyId).map(_.nonEmpty) } + def findSigningKeyIdByName(keyName: KeyName)(implicit + traceContext: TraceContext + ): EitherT[Future, CryptoPublicStoreError, Option[SigningPublicKey]] = + listSigningKeys.map(_.find(_.name.contains(keyName)).map(_.publicKey)) + + def findEncryptionKeyIdByName(keyName: KeyName)(implicit + traceContext: TraceContext + ): EitherT[Future, CryptoPublicStoreError, Option[EncryptionPublicKey]] = + listEncryptionKeys.map(_.find(_.name.contains(keyName)).map(_.publicKey)) + def publicKeysWithName(implicit traceContext: TraceContext ): EitherT[Future, CryptoPublicStoreError, Set[PublicKeyWithName]] = diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/store/db/DbCryptoPrivateStore.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/store/db/DbCryptoPrivateStore.scala index bf2e26cce..1ab6de2c7 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/store/db/DbCryptoPrivateStore.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/store/db/DbCryptoPrivateStore.scala @@ -11,10 +11,10 @@ import com.digitalasset.canton.crypto.store._ import com.digitalasset.canton.logging.NamedLoggerFactory import com.digitalasset.canton.metrics.MetricHandle.GaugeM import com.digitalasset.canton.metrics.TimedLoadGauge -import com.digitalasset.canton.resource.DbStorage.{DbAction, Profile} +import com.digitalasset.canton.resource.DbStorage.DbAction import com.digitalasset.canton.resource.{DbStorage, DbStore} import com.digitalasset.canton.tracing.TraceContext -import com.digitalasset.canton.util.{EitherTUtil, ErrorUtil} +import com.digitalasset.canton.util.EitherTUtil import io.functionmeta.functionFullName import slick.jdbc.{GetResult, SetParameter} @@ -36,23 +36,6 @@ class DbCryptoPrivateStore( private val queryTime: GaugeM[TimedLoadGauge, Double] = storage.metrics.loadGaugeM("crypto-private-store-query") - private val queryHmacSecret = - sql"select data from crypto_hmac_secret".as[HmacSecret].headOption - - private def insertHmacSecret(hmacSecret: HmacSecret): DbAction.WriteOnly[Int] = - storage.profile match { - case _: Profile.Oracle | _: Profile.H2 => - sqlu"""merge into crypto_hmac_secret using dual on (hmac_secret_id = 1) - when matched then update set data = $hmacSecret - when not matched then insert (data) values ($hmacSecret)""" - - case _: Profile.Postgres => - sqlu"""insert into crypto_hmac_secret(data) - values ($hmacSecret) - on conflict (hmac_secret_id) - do update set data = $hmacSecret""" - } - private def queryKeys[K: GetResult](purpose: KeyPurpose): DbAction.ReadOnly[Set[K]] = sql"select data, name from crypto_private_keys where purpose = $purpose" .as[K] @@ -148,41 +131,6 @@ class DbCryptoPrivateStore( ): EitherT[Future, CryptoPrivateStoreError, Unit] = insertKey[EncryptionPrivateKey, EncryptionPrivateKeyWithName](key, name) - override protected def writeHmacSecret( - hmacSecret: HmacSecret - )(implicit traceContext: TraceContext): EitherT[Future, CryptoPrivateStoreError, Unit] = { - - for { - nrRows <- EitherTUtil.fromFuture( - insertTime.metric.event(storage.update(insertHmacSecret(hmacSecret), functionFullName)), - err => CryptoPrivateStoreError.FailedToInsertHmacSecret(err.toString), - ) - _ <- nrRows match { - case 1 => EitherTUtil.unit[CryptoPrivateStoreError] - case 0 => - for { - existingHmacSecret <- loadHmacSecret().map( - _.getOrElse( - ErrorUtil.internalError( - new IllegalStateException("No existing HMAC secret found but failed to insert") - ) - ) - ) - _ <- EitherTUtil - .condUnitET[Future]( - existingHmacSecret == hmacSecret, - CryptoPrivateStoreError.HmacSecretAlreadyExists, - ) - .leftWiden[CryptoPrivateStoreError] - } yield () - case _ => - ErrorUtil.internalError( - new IllegalStateException(s"Updated more than 1 row for HMAC secrets: $nrRows") - ) - } - } yield () - } - override private[store] def listSigningKeys(implicit traceContext: TraceContext ): EitherT[Future, CryptoPrivateStoreError, Set[SigningPrivateKeyWithName]] = @@ -205,14 +153,6 @@ class DbCryptoPrivateStore( err => CryptoPrivateStoreError.FailedToListKeys(err.toString), ) - override private[store] def loadHmacSecret()(implicit - traceContext: TraceContext - ): EitherT[Future, CryptoPrivateStoreError, Option[HmacSecret]] = - EitherTUtil.fromFuture( - queryTime.metric.event(storage.query(queryHmacSecret, functionFullName)), - err => CryptoPrivateStoreError.FailedToLoadHmacSecret(err.toString), - ) - override protected def deletePrivateKey(keyId: Fingerprint)(implicit traceContext: TraceContext ): EitherT[Future, CryptoPrivateStoreError, Unit] = diff --git a/community/common/src/main/scala/com/digitalasset/canton/crypto/store/memory/InMemoryCryptoPrivateStore.scala b/community/common/src/main/scala/com/digitalasset/canton/crypto/store/memory/InMemoryCryptoPrivateStore.scala index 4e18b2376..b8bc66b8e 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/crypto/store/memory/InMemoryCryptoPrivateStore.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/crypto/store/memory/InMemoryCryptoPrivateStore.scala @@ -12,16 +12,10 @@ import com.digitalasset.canton.crypto.store.{ PrivateKeyWithName, SigningPrivateKeyWithName, } -import com.digitalasset.canton.crypto.{ - EncryptionPrivateKey, - Fingerprint, - HmacSecret, - SigningPrivateKey, -} +import com.digitalasset.canton.crypto.{EncryptionPrivateKey, Fingerprint, SigningPrivateKey} import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging} import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.util.TrieMapUtil -import java.util.concurrent.atomic.AtomicReference import com.digitalasset.canton.crypto.KeyName @@ -38,7 +32,6 @@ class InMemoryCryptoPrivateStore(override protected val loggerFactory: NamedLogg private val storedSigningKeyMap: TrieMap[Fingerprint, SigningPrivateKeyWithName] = TrieMap.empty private val storedDecryptionKeyMap: TrieMap[Fingerprint, EncryptionPrivateKeyWithName] = TrieMap.empty - private val storedHmacSecret: AtomicReference[Option[HmacSecret]] = new AtomicReference(None) private def errorDuplicate[K <: PrivateKeyWithName]( keyId: Fingerprint, @@ -83,12 +76,6 @@ class InMemoryCryptoPrivateStore(override protected val loggerFactory: NamedLogg .toEitherT } - override protected def writeHmacSecret( - hmacSecret: HmacSecret - )(implicit traceContext: TraceContext): EitherT[Future, CryptoPrivateStoreError, Unit] = { - EitherT.rightT(storedHmacSecret.set(Some(hmacSecret))) - } - override private[store] def listSigningKeys(implicit traceContext: TraceContext ): EitherT[Future, CryptoPrivateStoreError, Set[SigningPrivateKeyWithName]] = @@ -99,11 +86,6 @@ class InMemoryCryptoPrivateStore(override protected val loggerFactory: NamedLogg ): EitherT[Future, CryptoPrivateStoreError, Set[EncryptionPrivateKeyWithName]] = EitherT.rightT(storedDecryptionKeyMap.values.toSet) - override private[store] def loadHmacSecret()(implicit - traceContext: TraceContext - ): EitherT[Future, CryptoPrivateStoreError, Option[HmacSecret]] = - EitherT.rightT(storedHmacSecret.get()) - override protected def deletePrivateKey( keyId: Fingerprint )(implicit traceContext: TraceContext): EitherT[Future, CryptoPrivateStoreError, Unit] = { diff --git a/community/common/src/main/scala/com/digitalasset/canton/data/GenTransactionTree.scala b/community/common/src/main/scala/com/digitalasset/canton/data/GenTransactionTree.scala index 5f9a97e13..55bf01d2d 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/data/GenTransactionTree.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/data/GenTransactionTree.scala @@ -172,6 +172,7 @@ case class GenTransactionTree( def allLightTransactionViewTreesWithWitnessesAndSeeds( initSeed: SecureRandomness, hkdfOps: HkdfOps, + protocolVersion: ProtocolVersion, ): Either[HkdfError, Seq[(LightTransactionViewTree, Witnesses, SecureRandomness)]] = { val randomnessLength = initSeed.unwrap.size val witnessAndSeedMapE = @@ -191,7 +192,11 @@ case class GenTransactionTree( val viewIndex = tvt.viewPosition.position.headOption .getOrElse(throw new IllegalStateException("View with no position")) - val seedE = hkdfOps.hkdfExpand(parentSeed, randomnessLength, HkdfInfo.subview(viewIndex)) + val seedE = ProtocolCryptoApi.hkdf(hkdfOps, protocolVersion)( + parentSeed, + randomnessLength, + HkdfInfo.subview(viewIndex), + ) seedE.map(seed => ws.updated(tvt.viewPosition, witnesses -> seed)) } witnessAndSeedMapE.map { witnessAndSeedMap => diff --git a/community/common/src/main/scala/com/digitalasset/canton/environment/CantonNodeBootstrap.scala b/community/common/src/main/scala/com/digitalasset/canton/environment/CantonNodeBootstrap.scala index 8ad13c168..6d1e49652 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/environment/CantonNodeBootstrap.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/environment/CantonNodeBootstrap.scala @@ -13,6 +13,7 @@ import com.digitalasset.canton.config.{LocalNodeConfig, LocalNodeParameters, Pro import com.digitalasset.canton.crypto._ import com.digitalasset.canton.crypto.admin.grpc.GrpcVaultService import com.digitalasset.canton.crypto.admin.v0.VaultServiceGrpc +import com.digitalasset.canton.crypto.store.{CryptoPrivateStoreError, CryptoPublicStoreError} import com.digitalasset.canton.error.CantonError import com.digitalasset.canton.health.admin.data.NodeStatus import com.digitalasset.canton.health.admin.grpc.GrpcStatusService @@ -41,7 +42,7 @@ import com.digitalasset.canton.topology.store.{ TopologyStore, TopologyStoreFactory, } -import com.digitalasset.canton.topology.transaction._ +import com.digitalasset.canton.topology.transaction.{TopologyTransaction, _} import com.digitalasset.canton.tracing.TraceContext.withNewTraceContext import com.digitalasset.canton.tracing.{NoTracing, TraceContext, TracerProvider} import com.digitalasset.canton.util.retry @@ -349,7 +350,7 @@ abstract class CantonNodeBootstrapBase[ } yield () } - final def storeId(id: NodeId): EitherT[Future, String, Unit] = + final protected def storeId(id: NodeId): EitherT[Future, String, Unit] = for { previous <- EitherT.right(initializationStore.id) result <- previous match { @@ -394,25 +395,92 @@ abstract class CantonNodeBootstrapBase[ // utility functions used by automatic initialization of domain and participant protected def authorizeStateUpdate[E <: CantonError]( manager: TopologyManager[E], - key: PublicKey, + key: SigningPublicKey, mapping: TopologyStateUpdateMapping, )(implicit traceContext: TraceContext - ): EitherT[Future, String, SignedTopologyTransaction[TopologyChangeOp.Add]] = - manager - .authorize(TopologyStateUpdate.createAdd(mapping), Some(key.fingerprint), false) - .leftMap(_.toString) + ): EitherT[Future, String, Unit] = + authorizeIfNew(manager, TopologyStateUpdate.createAdd(mapping), key) + + private def authorizeIfNew[E <: CantonError, Op <: TopologyChangeOp]( + manager: TopologyManager[E], + transaction: TopologyTransaction[Op], + signingKey: SigningPublicKey, + )(implicit + traceContext: TraceContext + ): EitherT[Future, String, Unit] = for { + exists <- EitherT.right( + manager.signedMappingAlreadyExists(transaction.element.mapping, signingKey.fingerprint) + ) + res <- + if (exists) { + logger.debug(s"Skipping existing ${transaction.element.mapping}") + EitherT.rightT[Future, String](()) + } else + manager + .authorize(transaction, Some(signingKey.fingerprint), false) + .leftMap(_.toString) + .map(_ => ()) + } yield res protected def authorizeDomainGovernance[E <: CantonError]( manager: TopologyManager[E], - key: PublicKey, + key: SigningPublicKey, mapping: DomainGovernanceMapping, )(implicit traceContext: TraceContext - ): EitherT[Future, String, SignedTopologyTransaction[TopologyChangeOp.Replace]] = - manager - .authorize(DomainGovernanceTransaction(mapping), Some(key.fingerprint), false) - .leftMap(_.toString) + ): EitherT[Future, String, Unit] = + authorizeIfNew(manager, DomainGovernanceTransaction(mapping), key) + + protected def getOrCreateSigningKey( + name: String + )(implicit traceContext: TraceContext): EitherT[Future, String, SigningPublicKey] = + getOrCreateKey( + "signing", + crypto.cryptoPublicStore.findSigningKeyIdByName, + name => crypto.generateSigningKey(name = name).leftMap(_.toString), + crypto.cryptoPrivateStore.existsSigningKey, + name, + ) + + protected def getOrCreateEncryptionKey( + name: String + )(implicit traceContext: TraceContext): EitherT[Future, String, EncryptionPublicKey] = + getOrCreateKey( + "encryption", + crypto.cryptoPublicStore.findEncryptionKeyIdByName, + name => crypto.generateEncryptionKey(name = name).leftMap(_.toString), + crypto.cryptoPrivateStore.existsPrivateKey, + name, + ) + + private def getOrCreateKey[P <: PublicKey]( + typ: String, + findPubKeyIdByName: KeyName => EitherT[Future, CryptoPublicStoreError, Option[P]], + generateKey: Option[KeyName] => EitherT[Future, String, P], + existPrivateKeyByFp: Fingerprint => EitherT[Future, CryptoPrivateStoreError, Boolean], + name: String, + ): EitherT[Future, String, P] = for { + keyName <- EitherT.fromEither[Future](KeyName.create(name)) + keyIdO <- findPubKeyIdByName(keyName) + .leftMap(err => s"Failure while looking for $typ key $name in public store: ${err}") + pubKey <- keyIdO.fold( + generateKey(Some(keyName)) + .leftMap(err => s"Failure while generating $typ key for $name: $err") + ) { keyWithName => + val fingerprint = keyWithName.fingerprint + existPrivateKeyByFp(fingerprint) + .leftMap(err => + s"Failure while looking for $typ key $fingerprint of $name in private key store: $err" + ) + .transform { + case Right(true) => Right(keyWithName) + case Right(false) => + Left(s"Broken private key store: Could not find $typ key $fingerprint of $name") + case Left(err) => Left(err) + } + } + } yield pubKey /** Poll the datastore to see if the id has been initialized in case a replica initializes the node */ private def waitForReplicaInitialization(): Unit = blocking { diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/DomainParameters.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/DomainParameters.scala index ebd002b71..3685e6708 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/DomainParameters.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/DomainParameters.scala @@ -309,7 +309,6 @@ object DynamicDomainParameters extends HasVersionedMessageCompanion[DynamicDomai case _: RemoteClock | _: SimClock => defaultTopologyChangeDelayNonStandardClock case _ => defaultTopologyChangeDelay } - initialValues(topologyChangeDelay) } diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/AcsCommitment.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/AcsCommitment.scala index 222ed3e39..55061a312 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/AcsCommitment.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/AcsCommitment.scala @@ -17,9 +17,9 @@ import com.digitalasset.canton.store.db.DbDeserializationException import com.digitalasset.canton.time.PositiveSeconds import com.digitalasset.canton.util.NoCopy import com.digitalasset.canton.version.{ - HasMemoizedVersionedMessageCompanion, + HasMemoizedProtocolVersionedWrapperCompanion, HasProtoV0, - HasVersionedWrapper, + HasProtocolVersionedWrapper, ProtocolVersion, VersionedMessage, } @@ -122,16 +122,16 @@ case class AcsCommitment private ( counterParticipant: ParticipantId, period: CommitmentPeriod, commitment: AcsCommitment.CommitmentType, -)(override val deserializedFrom: Option[ByteString]) - extends HasVersionedWrapper[VersionedMessage[AcsCommitment]] +)( + val representativeProtocolVersion: ProtocolVersion, + override val deserializedFrom: Option[ByteString], +) extends HasProtocolVersionedWrapper[VersionedMessage[AcsCommitment]] with HasProtoV0[v0.AcsCommitment] with SignedProtocolMessageContent with NoCopy { - override protected def toProtoVersioned( - version: ProtocolVersion - ): VersionedMessage[AcsCommitment] = - VersionedMessage(toProtoV0.toByteString, 0) + override protected def toProtoVersioned: VersionedMessage[AcsCommitment] = + AcsCommitment.toProtoVersioned(this) override protected def toProtoV0: v0.AcsCommitment = { v0.AcsCommitment( @@ -144,8 +144,8 @@ case class AcsCommitment private ( ) } - override protected[this] def toByteStringUnmemoized(version: ProtocolVersion): ByteString = - super[HasVersionedWrapper].toByteString(version) + override protected[this] def toByteStringUnmemoized: ByteString = + super[HasProtocolVersionedWrapper].toByteString protected[messages] def toProtoSomeSignedProtocolMessage : v0.SignedProtocolMessage.SomeSignedProtocolMessage = @@ -164,11 +164,19 @@ case class AcsCommitment private ( } } -object AcsCommitment extends HasMemoizedVersionedMessageCompanion[AcsCommitment] { +object AcsCommitment extends HasMemoizedProtocolVersionedWrapperCompanion[AcsCommitment] { override val name: String = "AcsCommitment" - val supportedProtoVersions: Map[Int, Parser] = Map( - 0 -> supportedProtoVersionMemoized(v0.AcsCommitment)(fromProtoV0) + val supportedProtoVersions = SupportedProtoVersions( + 0 -> VersionedProtoConverter( + ProtocolVersion.v2_0_0, + supportedProtoVersionMemoized(v0.AcsCommitment)(fromProtoV0), + _.toProtoV0.toByteString, + ) + ) + + private val protocolVersionRepresentative = protocolVersionRepresentativeFor( + ProtocolVersion.v2_0_0_Todo_i8793 ) type CommitmentType = ByteString @@ -193,7 +201,10 @@ object AcsCommitment extends HasMemoizedVersionedMessageCompanion[AcsCommitment] sender: ParticipantId, timestamp: CantonTimestamp, commitment: CommitmentType, - )(deserializedFrom: Option[ByteString]): AcsCommitment = + )( + representativeProtocolVersion: ProtocolVersion, + deserializedFrom: Option[ByteString], + ): AcsCommitment = throw new UnsupportedOperationException("Use the create/tryCreate methods instead") def create( @@ -202,8 +213,12 @@ object AcsCommitment extends HasMemoizedVersionedMessageCompanion[AcsCommitment] counterParticipant: ParticipantId, period: CommitmentPeriod, commitment: CommitmentType, + protocolVersion: ProtocolVersion, ): AcsCommitment = - new AcsCommitment(domainId, sender, counterParticipant, period, commitment)(None) + new AcsCommitment(domainId, sender, counterParticipant, period, commitment)( + protocolVersion, + None, + ) private def fromProtoV0(protoMsg: v0.AcsCommitment)( bytes: ByteString @@ -236,7 +251,10 @@ object AcsCommitment extends HasMemoizedVersionedMessageCompanion[AcsCommitment] period = CommitmentPeriod(fromExclusive, periodLength) cmt = protoMsg.commitment commitment = commitmentTypeFromByteString(cmt) - } yield new AcsCommitment(domainId, sender, counterParticipant, period, commitment)(Some(bytes)) + } yield new AcsCommitment(domainId, sender, counterParticipant, period, commitment)( + protocolVersionRepresentativeFor(0), + Some(bytes), + ) } implicit val acsCommitmentCast: SignedMessageContentCast[AcsCommitment] = { @@ -251,7 +269,10 @@ object AcsCommitment extends HasMemoizedVersionedMessageCompanion[AcsCommitment] GetResult[CommitmentPeriod], GetResult[CommitmentType], ).andThen { case (sender, counterParticipant, period, commitment) => - AcsCommitment(domainId, sender, counterParticipant, period, commitment)(None) + AcsCommitment(domainId, sender, counterParticipant, period, commitment)( + protocolVersionRepresentative, + None, + ) } } diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/EncryptedViewMessage.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/EncryptedViewMessage.scala index 526393fd8..b76922fcc 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/EncryptedViewMessage.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/EncryptedViewMessage.scala @@ -260,6 +260,7 @@ object EncryptedViewMessage { snapshot: DomainSnapshotSyncCryptoApi, encrypted: EncryptedViewMessage[VT], participantId: ParticipantId, + protocolVersion: ProtocolVersion, optViewRandomness: Option[SecureRandomness] = None, )(deserialize: ByteString => Either[DeserializationError, encrypted.encryptedView.viewType.View])( implicit ec: ExecutionContext @@ -280,11 +281,12 @@ object EncryptedViewMessage { viewRandomness <- optViewRandomness.fold( decryptRandomness(snapshot, encrypted, participantId) )(r => EitherT.pure(r)) - viewKey <- eitherT( - pureCrypto - .hkdfExpand(viewRandomness, keyLength, HkdfInfo.ViewKey) - .leftMap(HkdfExpansionError(_, encrypted)) - ) + viewKey <- + eitherT( + ProtocolCryptoApi + .hkdf(pureCrypto, protocolVersion)(viewRandomness, keyLength, HkdfInfo.ViewKey) + .leftMap(HkdfExpansionError(_, encrypted)) + ) decrypted <- eitherT( EncryptedView .decrypt(pureCrypto, viewKey, encrypted.encryptedView)(deserialize) diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/InformeeMessage.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/InformeeMessage.scala index e97ecd4d0..6eb49df55 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/InformeeMessage.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/InformeeMessage.scala @@ -42,11 +42,13 @@ case class InformeeMessage(fullInformeeTree: FullInformeeTree) requestId: RequestId, verdict: Verdict, recipientParties: Set[LfPartyId], + protocolVersion: ProtocolVersion, ): TransactionResultMessage = TransactionResultMessage( requestId, verdict, fullInformeeTree.informeeTreeUnblindedFor(recipientParties), + protocolVersion, ) // Implementing a `toProto` method allows us to compose serializable classes. diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MalformedMediatorRequestResult.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MalformedMediatorRequestResult.scala index e8d159e3d..dbfe63f26 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MalformedMediatorRequestResult.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MalformedMediatorRequestResult.scala @@ -14,9 +14,9 @@ import com.digitalasset.canton.serialization.ProtoConverter.ParsingResult import com.digitalasset.canton.topology.DomainId import com.digitalasset.canton.util.NoCopy import com.digitalasset.canton.version.{ - HasMemoizedVersionedMessageCompanion, + HasMemoizedProtocolVersionedWrapperCompanion, HasProtoV0, - HasVersionedWrapper, + HasProtocolVersionedWrapper, ProtocolVersion, VersionedMessage, } @@ -34,10 +34,10 @@ case class MalformedMediatorRequestResult private ( override val domainId: DomainId, override val viewType: ViewType, override val verdict: Verdict.MediatorReject, -)(val deserializedFrom: Option[ByteString]) +)(val representativeProtocolVersion: ProtocolVersion, val deserializedFrom: Option[ByteString]) extends MediatorResult with SignedProtocolMessageContent - with HasVersionedWrapper[VersionedMessage[MalformedMediatorRequestResult]] + with HasProtocolVersionedWrapper[VersionedMessage[MalformedMediatorRequestResult]] with HasProtoV0[v0.MalformedMediatorRequestResult] with NoCopy with PrettyPrinting { @@ -50,9 +50,8 @@ case class MalformedMediatorRequestResult private ( getCryptographicEvidence ) - override protected def toProtoVersioned( - version: ProtocolVersion - ): VersionedMessage[MalformedMediatorRequestResult] = VersionedMessage(toProtoV0.toByteString, 0) + override protected def toProtoVersioned: VersionedMessage[MalformedMediatorRequestResult] = + MalformedMediatorRequestResult.toProtoVersioned(this) override protected def toProtoV0: v0.MalformedMediatorRequestResult = v0.MalformedMediatorRequestResult( @@ -62,8 +61,8 @@ case class MalformedMediatorRequestResult private ( rejection = Some(verdict.toProtoMediatorRejectV0), ) - override protected[this] def toByteStringUnmemoized(version: ProtocolVersion): ByteString = - super[HasVersionedWrapper].toByteString(version) + override protected[this] def toByteStringUnmemoized: ByteString = + super[HasProtocolVersionedWrapper].toByteString override def pretty: Pretty[MalformedMediatorRequestResult] = prettyOfClass( param("request id", _.requestId), @@ -74,11 +73,15 @@ case class MalformedMediatorRequestResult private ( } object MalformedMediatorRequestResult - extends HasMemoizedVersionedMessageCompanion[MalformedMediatorRequestResult] { + extends HasMemoizedProtocolVersionedWrapperCompanion[MalformedMediatorRequestResult] { override val name: String = "MalformedMediatorRequestResult" - val supportedProtoVersions: Map[Int, Parser] = Map( - 0 -> supportedProtoVersionMemoized(v0.MalformedMediatorRequestResult)(fromProtoV0) + val supportedProtoVersions = SupportedProtoVersions( + 0 -> VersionedProtoConverter( + ProtocolVersion.v2_0_0, + supportedProtoVersionMemoized(v0.MalformedMediatorRequestResult)(fromProtoV0), + _.toProtoV0.toByteString, + ) ) private def apply( @@ -86,7 +89,10 @@ object MalformedMediatorRequestResult domainId: DomainId, viewType: ViewType, verdict: Verdict.MediatorReject, - )(deserializedFrom: Option[ByteString]): MalformedMediatorRequestResult = + )( + representativeProtocolVersion: ProtocolVersion, + deserializedFrom: Option[ByteString], + ): MalformedMediatorRequestResult = throw new UnsupportedOperationException("Use the other factory methods instead") def apply( @@ -94,8 +100,12 @@ object MalformedMediatorRequestResult domainId: DomainId, viewType: ViewType, verdict: Verdict.MediatorReject, + protocolVersion: ProtocolVersion, ): MalformedMediatorRequestResult = - new MalformedMediatorRequestResult(requestId, domainId, viewType, verdict)(None) + new MalformedMediatorRequestResult(requestId, domainId, viewType, verdict)( + protocolVersionRepresentativeFor(protocolVersion), + None, + ) private def fromProtoV0(protoResultMsg: v0.MalformedMediatorRequestResult)( bytes: ByteString @@ -111,7 +121,10 @@ object MalformedMediatorRequestResult domainId <- DomainId.fromProtoPrimitive(domainIdP, "domain_id") viewType <- ViewType.fromProtoEnum(viewTypeP) reject <- ProtoConverter.parseRequired(MediatorReject.fromProtoV0, "rejection", rejectP) - } yield new MalformedMediatorRequestResult(requestId, domainId, viewType, reject)(Some(bytes)) + } yield new MalformedMediatorRequestResult(requestId, domainId, viewType, reject)( + protocolVersionRepresentativeFor(0), + Some(bytes), + ) } implicit val malformedMediatorRequestResultCast diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorRequest.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorRequest.scala index 57af8a3d8..2941656f3 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorRequest.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorRequest.scala @@ -10,6 +10,7 @@ import com.digitalasset.canton.data.{Informee, ViewType} import com.digitalasset.canton.protocol.messages.ProtocolMessage.ProtocolMessageContentCast import com.digitalasset.canton.protocol.{ConfirmationPolicy, RequestId, RootHash, ViewHash} import com.digitalasset.canton.topology.MediatorId +import com.digitalasset.canton.version.ProtocolVersion trait MediatorRequest extends ProtocolMessage { def requestUuid: UUID @@ -30,6 +31,7 @@ trait MediatorRequest extends ProtocolMessage { requestId: RequestId, verdict: Verdict, recipientParties: Set[LfPartyId], + protocolVersion: ProtocolVersion, ): MediatorResult with SignedProtocolMessageContent def confirmationPolicy: ConfirmationPolicy diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorResponse.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorResponse.scala index 1d411acf8..4b0cdd1b2 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorResponse.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorResponse.scala @@ -16,9 +16,9 @@ import com.digitalasset.canton.serialization.ProtoConverter import com.digitalasset.canton.serialization.ProtoConverter.ParsingResult import com.digitalasset.canton.util.NoCopy import com.digitalasset.canton.version.{ - HasMemoizedVersionedMessageCompanion, + HasMemoizedProtocolVersionedWrapperCompanion, HasProtoV0, - HasVersionedWrapper, + HasProtocolVersionedWrapper, ProtocolVersion, VersionedMessage, } @@ -53,9 +53,11 @@ case class MediatorResponse private ( rootHash: Option[RootHash], confirmingParties: Set[LfPartyId], override val domainId: DomainId, -)(override val deserializedFrom: Option[ByteString]) - extends SignedProtocolMessageContent - with HasVersionedWrapper[VersionedMessage[MediatorResponse]] +)( + val representativeProtocolVersion: ProtocolVersion, + override val deserializedFrom: Option[ByteString], +) extends SignedProtocolMessageContent + with HasProtocolVersionedWrapper[VersionedMessage[MediatorResponse]] with HasProtoV0[v0.MediatorResponse] with HasDomainId with NoCopy { @@ -77,13 +79,11 @@ case class MediatorResponse private ( throw InvalidMediatorResponse(show"View mash must not be empty for verdict $localVerdict") } - protected override def toByteStringUnmemoized(version: ProtocolVersion): ByteString = - super[HasVersionedWrapper].toByteString(version) + protected override def toByteStringUnmemoized: ByteString = + super[HasProtocolVersionedWrapper].toByteString - override protected def toProtoVersioned( - version: ProtocolVersion - ): VersionedMessage[MediatorResponse] = - VersionedMessage(toProtoV0.toByteString, 0) + override protected def toProtoVersioned: VersionedMessage[MediatorResponse] = + MediatorResponse.toProtoVersioned(this) override protected def toProtoV0: v0.MediatorResponse = v0.MediatorResponse( @@ -103,11 +103,15 @@ case class MediatorResponse private ( override def hashPurpose: HashPurpose = HashPurpose.MediatorResponseSignature } -object MediatorResponse extends HasMemoizedVersionedMessageCompanion[MediatorResponse] { +object MediatorResponse extends HasMemoizedProtocolVersionedWrapperCompanion[MediatorResponse] { override val name: String = "MediatorResponse" - val supportedProtoVersions: Map[Int, Parser] = Map( - 0 -> supportedProtoVersionMemoized(v0.MediatorResponse)(fromProtoV0) + val supportedProtoVersions = SupportedProtoVersions( + 0 -> VersionedProtoConverter( + ProtocolVersion.v2_0_0, + supportedProtoVersionMemoized(v0.MediatorResponse)(fromProtoV0), + _.toProtoV0.toByteString, + ) ) case class InvalidMediatorResponse(msg: String) extends RuntimeException(msg) @@ -122,7 +126,7 @@ object MediatorResponse extends HasMemoizedVersionedMessageCompanion[MediatorRes rootHash: Option[RootHash], confirmingParties: Set[LfPartyId], domainId: DomainId, - )(deserializedFrom: Option[ByteString]) = + )(representativeProtocolVersion: ProtocolVersion, deserializedFrom: Option[ByteString]) = throw new UnsupportedOperationException("Use the public apply method") // Variant of "tryCreate" that returns Left(...) instead of throwing an exception. @@ -137,9 +141,19 @@ object MediatorResponse extends HasMemoizedVersionedMessageCompanion[MediatorRes rootHash: Option[RootHash], confirmingParties: Set[LfPartyId], domainId: DomainId, + protocolVersion: ProtocolVersion, ): Either[InvalidMediatorResponse, MediatorResponse] = Either.catchOnly[InvalidMediatorResponse]( - tryCreate(requestId, sender, viewHash, localVerdict, rootHash, confirmingParties, domainId) + tryCreate( + requestId, + sender, + viewHash, + localVerdict, + rootHash, + confirmingParties, + domainId, + protocolVersionRepresentativeFor(protocolVersion), + ) ) // This method is tailored to the case that the caller already knows that the parameters meet the object invariants. @@ -163,6 +177,7 @@ object MediatorResponse extends HasMemoizedVersionedMessageCompanion[MediatorRes rootHash: Option[RootHash], confirmingParties: Set[LfPartyId], domainId: DomainId, + protocolVersion: ProtocolVersion, ): MediatorResponse = new MediatorResponse( requestId, @@ -172,7 +187,7 @@ object MediatorResponse extends HasMemoizedVersionedMessageCompanion[MediatorRes rootHash, confirmingParties, domainId, - )(None) + )(protocolVersionRepresentativeFor(protocolVersion), None) private def fromProtoV0(mediatorResponseP: v0.MediatorResponse)( bytes: ByteString @@ -210,7 +225,7 @@ object MediatorResponse extends HasMemoizedVersionedMessageCompanion[MediatorRes rootHashO, confirmingParties.toSet, domainId, - )(Some(bytes)) + )(supportedProtoVersions.protocolVersionRepresentativeFor(protoVersion = 0), Some(bytes)) ) .leftMap(err => InvariantViolation(err.toString)) } yield response diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorResult.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorResult.scala index 3c9c6bcbe..c294252a9 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorResult.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/MediatorResult.scala @@ -5,9 +5,9 @@ package com.digitalasset.canton.protocol.messages import com.digitalasset.canton.data.ViewType import com.digitalasset.canton.protocol.messages.SignedProtocolMessageContent.SignedMessageContentCast -import com.digitalasset.canton.serialization.MemoizedEvidence +import com.digitalasset.canton.serialization.ProtocolVersionedMemoizedEvidence -trait MediatorResult extends MemoizedEvidence with HasDomainId with HasRequestId { +trait MediatorResult extends ProtocolVersionedMemoizedEvidence with HasDomainId with HasRequestId { def verdict: Verdict def viewType: ViewType diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/SignedProtocolMessageContent.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/SignedProtocolMessageContent.scala index 31a300635..4e653927c 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/SignedProtocolMessageContent.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/SignedProtocolMessageContent.scala @@ -6,16 +6,16 @@ package com.digitalasset.canton.protocol.messages import com.digitalasset.canton.crypto.HashPurpose import com.digitalasset.canton.logging.pretty.{Pretty, PrettyPrinting} import com.digitalasset.canton.protocol.v0 -import com.digitalasset.canton.serialization.MemoizedEvidence +import com.digitalasset.canton.serialization.ProtocolVersionedMemoizedEvidence trait SignedProtocolMessageContent - extends MemoizedEvidence + extends ProtocolVersionedMemoizedEvidence with HasDomainId with PrettyPrinting with Product with Serializable { - /** Converts this object into a [[com.google.protobuf.ByteString]] using [[com.digitalasset.canton.serialization.MemoizedEvidence.getCryptographicEvidence]] + /** Converts this object into a [[com.google.protobuf.ByteString]] using [[com.digitalasset.canton.serialization.ProtocolVersionedMemoizedEvidence.getCryptographicEvidence]] * and wraps the result in the appropriate [[com.digitalasset.canton.protocol.v0.SignedProtocolMessage.SomeSignedProtocolMessage]] constructor. */ protected[messages] def toProtoSomeSignedProtocolMessage diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TranferOutMediatorMessage.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TranferOutMediatorMessage.scala index 2911ecfed..f52349edb 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TranferOutMediatorMessage.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TranferOutMediatorMessage.scala @@ -49,13 +49,20 @@ case class TransferOutMediatorMessage(tree: TransferOutViewTree) requestId: RequestId, verdict: Verdict, recipientParties: Set[LfPartyId], + protocolVersion: ProtocolVersion, ): MediatorResult with SignedProtocolMessageContent = { val informees = commonData.stakeholders ++ commonData.adminParties require( recipientParties.subsetOf(informees), "Recipient parties of the transfer-out result are neither stakeholders nor admin parties", ) - TransferResult.create(requestId, informees, TransferOutDomainId(domainId), verdict) + TransferResult.create( + requestId, + informees, + TransferOutDomainId(domainId), + verdict, + protocolVersion, + ) } override def toProtoV0: v0.TransferOutMediatorMessage = diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransactionResultMessage.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransactionResultMessage.scala index ec95fba2b..3eac12e73 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransactionResultMessage.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransactionResultMessage.scala @@ -15,9 +15,9 @@ import com.digitalasset.canton.serialization.ProtoConverter.ParsingResult import com.digitalasset.canton.topology.DomainId import com.digitalasset.canton.util.NoCopy import com.digitalasset.canton.version.{ - HasMemoizedVersionedMessageWithContextCompanion, + HasMemoizedProtocolVersionedWithContextCompanion, HasProtoV0, - HasVersionedWrapper, + HasProtocolVersionedWrapper, ProtocolVersion, VersionedMessage, } @@ -35,10 +35,10 @@ case class TransactionResultMessage private ( override val requestId: RequestId, override val verdict: Verdict, notificationTree: InformeeTree, -)(val deserializedFrom: Option[ByteString]) +)(val representativeProtocolVersion: ProtocolVersion, val deserializedFrom: Option[ByteString]) extends RegularMediatorResult with NoCopy - with HasVersionedWrapper[VersionedMessage[TransactionResultMessage]] + with HasProtocolVersionedWrapper[VersionedMessage[TransactionResultMessage]] with HasProtoV0[v0.TransactionResultMessage] with PrettyPrinting { @@ -53,12 +53,11 @@ case class TransactionResultMessage private ( * Must meet the contract of [[com.digitalasset.canton.serialization.HasCryptographicEvidence.getCryptographicEvidence]] * except that when called several times, different [[com.google.protobuf.ByteString]]s may be returned. */ - override protected[this] def toByteStringUnmemoized(version: ProtocolVersion): ByteString = - super[HasVersionedWrapper].toByteString(version) + override protected[this] def toByteStringUnmemoized: ByteString = + super[HasProtocolVersionedWrapper].toByteString - override protected def toProtoVersioned( - version: ProtocolVersion - ): VersionedMessage[TransactionResultMessage] = VersionedMessage(toProtoV0.toByteString, 0) + override protected def toProtoVersioned: VersionedMessage[TransactionResultMessage] = + TransactionResultMessage.toProtoVersioned(this) override protected def toProtoV0: v0.TransactionResultMessage = v0.TransactionResultMessage( @@ -83,20 +82,25 @@ case class TransactionResultMessage private ( } object TransactionResultMessage - extends HasMemoizedVersionedMessageWithContextCompanion[ + extends HasMemoizedProtocolVersionedWithContextCompanion[ TransactionResultMessage, HashOps, ] { override val name: String = "TransactionResultMessage" - val supportedProtoVersions: Map[Int, Parser] = Map( - 0 -> supportedProtoVersionMemoized(v0.TransactionResultMessage) { case (hashOps, proto) => - fromProtoV0(proto, hashOps) - } + val supportedProtoVersions = SupportedProtoVersions( + 0 -> VersionedProtoConverter( + ProtocolVersion.v2_0_0, + supportedProtoVersionMemoized(v0.TransactionResultMessage) { case (hashOps, proto) => + fromProtoV0(proto, hashOps) + }, + _.toProtoV0.toByteString, + ) ) private[this] def apply(requestId: RequestId, verdict: Verdict, notificationTree: InformeeTree)( - deseializedFrom: Option[ByteString] + representativeProtocolVersion: ProtocolVersion, + deseializedFrom: Option[ByteString], ): TransactionResultMessage = throw new UnsupportedOperationException("Use the public apply method instead") @@ -104,8 +108,12 @@ object TransactionResultMessage requestId: RequestId, verdict: Verdict, notificationTree: InformeeTree, + protocolVersion: ProtocolVersion, ): TransactionResultMessage = - new TransactionResultMessage(requestId, verdict, notificationTree)(None) + new TransactionResultMessage(requestId, verdict, notificationTree)( + protocolVersionRepresentativeFor(protocolVersion), + None, + ) private def fromProtoV0(protoResultMessage: v0.TransactionResultMessage, hashOps: HashOps)( bytes: ByteString @@ -123,7 +131,8 @@ object TransactionResultMessage .leftWiden[ProtoDeserializationError] notificationTree <- InformeeTree.fromProtoV0(hashOps, protoNotificationTree) } yield new TransactionResultMessage(requestId, transactionResult, notificationTree)( - Some(bytes) + protocolVersionRepresentativeFor(0), + Some(bytes), ) implicit val transactionResultMessageCast: SignedMessageContentCast[TransactionResultMessage] = { diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransferInMediatorMessage.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransferInMediatorMessage.scala index 2b015f167..accfb9164 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransferInMediatorMessage.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransferInMediatorMessage.scala @@ -49,13 +49,20 @@ case class TransferInMediatorMessage(tree: TransferInViewTree) requestId: RequestId, verdict: Verdict, recipientParties: Set[LfPartyId], + protocolVersion: ProtocolVersion, ): MediatorResult with SignedProtocolMessageContent = { val informees = commonData.stakeholders require( recipientParties.subsetOf(informees), "Recipient parties of the transfer-in result must be stakeholders.", ) - TransferResult.create(requestId, informees, TransferInDomainId(domainId), verdict) + TransferResult.create( + requestId, + informees, + TransferInDomainId(domainId), + verdict, + protocolVersion, + ) } override def toProtoEnvelopeContentV0(version: ProtocolVersion): v0.EnvelopeContent = diff --git a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransferResult.scala b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransferResult.scala index a140659df..665ae4a6c 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransferResult.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/protocol/messages/TransferResult.scala @@ -21,9 +21,9 @@ import com.digitalasset.canton.serialization.ProtoConverter import com.digitalasset.canton.serialization.ProtoConverter.ParsingResult import com.digitalasset.canton.util.NoCopy import com.digitalasset.canton.version.{ - HasMemoizedVersionedMessageCompanion, + HasMemoizedProtocolVersionedWrapperCompanion, HasProtoV0, - HasVersionedWrapper, + HasProtocolVersionedWrapper, ProtocolVersion, VersionedMessage, } @@ -40,9 +40,11 @@ case class TransferResult[+Domain <: TransferDomainId] private ( informees: Set[LfPartyId], domain: Domain, // For transfer-out, this is the origin domain. For transfer-in, this is the target domain. override val verdict: Verdict, -)(override val deserializedFrom: Option[ByteString]) - extends RegularMediatorResult - with HasVersionedWrapper[VersionedMessage[TransferResult[Domain]]] +)( + val representativeProtocolVersion: ProtocolVersion, + override val deserializedFrom: Option[ByteString], +) extends RegularMediatorResult + with HasProtocolVersionedWrapper[VersionedMessage[TransferResult[TransferDomainId]]] with HasProtoV0[v0.TransferResult] with NoCopy with PrettyPrinting { @@ -59,10 +61,8 @@ case class TransferResult[+Domain <: TransferDomainId] private ( : v0.SignedProtocolMessage.SomeSignedProtocolMessage.TransferResult = v0.SignedProtocolMessage.SomeSignedProtocolMessage.TransferResult(getCryptographicEvidence) - override def toProtoVersioned( - version: ProtocolVersion - ): VersionedMessage[TransferResult[Domain]] = - VersionedMessage(toProtoV0.toByteString, 0) + override def toProtoVersioned: VersionedMessage[TransferResult[TransferDomainId]] = + TransferResult.toProtoVersioned(this) override def toProtoV0: v0.TransferResult = { val domainP = (domain: @unchecked) match { @@ -79,8 +79,8 @@ case class TransferResult[+Domain <: TransferDomainId] private ( ) } - override protected[this] def toByteStringUnmemoized(version: ProtocolVersion): ByteString = - super[HasVersionedWrapper].toByteString(version) + override protected[this] def toByteStringUnmemoized: ByteString = + super[HasProtocolVersionedWrapper].toByteString override def hashPurpose: HashPurpose = HashPurpose.TransferResultSignature @@ -91,8 +91,18 @@ case class TransferResult[+Domain <: TransferDomainId] private ( F.map(f(domain)) { newDomain => if (newDomain eq domain) this.asInstanceOf[TransferResult[Domain2]] else if (newDomain == domain) - new TransferResult(requestId, informees, newDomain, verdict)(deserializedFrom) - else TransferResult.create(requestId, informees, newDomain, verdict) + new TransferResult(requestId, informees, newDomain, verdict)( + representativeProtocolVersion, + deserializedFrom, + ) + else + TransferResult.create( + requestId, + informees, + newDomain, + verdict, + representativeProtocolVersion, + ) } override def pretty: Pretty[TransferResult[_ <: TransferDomainId]] = @@ -105,13 +115,15 @@ case class TransferResult[+Domain <: TransferDomainId] private ( } object TransferResult - extends HasMemoizedVersionedMessageCompanion[ - TransferResult[TransferDomainId], - ] { + extends HasMemoizedProtocolVersionedWrapperCompanion[TransferResult[TransferDomainId]] { override val name: String = "TransferResult" - val supportedProtoVersions: Map[Int, Parser] = Map( - 0 -> supportedProtoVersionMemoized(v0.TransferResult)(fromProtoV0) + val supportedProtoVersions = SupportedProtoVersions( + 0 -> VersionedProtoConverter( + ProtocolVersion.v2_0_0, + supportedProtoVersionMemoized(v0.TransferResult)(fromProtoV0), + _.toProtoV0.toByteString, + ) ) private def apply[Domain <: TransferDomainId]( @@ -120,7 +132,8 @@ object TransferResult domain: Domain, verdict: Verdict, )( - deserializedFrom: Option[ByteString] + representativeProtocolVersion: ProtocolVersion, + deserializedFrom: Option[ByteString], ): TransferResult[Domain] = throw new UnsupportedOperationException( "Use the create method instead" ) @@ -130,8 +143,9 @@ object TransferResult informees: Set[LfPartyId], domain: Domain, verdict: Verdict, + protocolVersion: ProtocolVersion, ) = - new TransferResult[Domain](requestId, informees, domain, verdict)(None) + new TransferResult[Domain](requestId, informees, domain, verdict)(protocolVersion, None) private def fromProtoV0(transferResultP: v0.TransferResult)( bytes: ByteString @@ -159,7 +173,10 @@ object TransferResult verdict <- ProtoConverter .required("TransferResult.verdict", maybeVerdictP) .flatMap(Verdict.fromProtoV0) - } yield new TransferResult(requestId, informees.toSet, domain, verdict)(Some(bytes)) + } yield new TransferResult(requestId, informees.toSet, domain, verdict)( + protocolVersionRepresentativeFor(0), + Some(bytes), + ) } implicit def transferResultCast[Kind <: TransferDomainId](implicit diff --git a/community/common/src/main/scala/com/digitalasset/canton/sequencing/authentication/AuthenticationToken.scala b/community/common/src/main/scala/com/digitalasset/canton/sequencing/authentication/AuthenticationToken.scala index 73da5b5b2..ac109fcec 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/sequencing/authentication/AuthenticationToken.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/sequencing/authentication/AuthenticationToken.scala @@ -6,7 +6,7 @@ package com.digitalasset.canton.sequencing.authentication import cats.syntax.either._ import com.digitalasset.canton.checked import com.digitalasset.canton.config.RequireTypes.String300 -import com.digitalasset.canton.crypto.SecureRandomness +import com.digitalasset.canton.crypto.RandomOps import com.digitalasset.canton.serialization.{DeserializationError, HasCryptographicEvidence} import com.digitalasset.canton.store.db.{DbDeserializationException, DbSerializationException} import com.digitalasset.canton.util.{HexString, NoCopy} @@ -36,8 +36,8 @@ object AuthenticationToken { private[this] def apply(bytes: ByteString): AuthenticationToken = throw new UnsupportedOperationException("Use the generate methods instead") - def generate(): AuthenticationToken = { - new AuthenticationToken(SecureRandomness.randomByteString(length)) + def generate(randomOps: RandomOps): AuthenticationToken = { + new AuthenticationToken(randomOps.generateRandomByteString(length)) } def fromProtoPrimitive(bytes: ByteString): Either[DeserializationError, AuthenticationToken] = diff --git a/community/common/src/main/scala/com/digitalasset/canton/sequencing/client/SequencerClient.scala b/community/common/src/main/scala/com/digitalasset/canton/sequencing/client/SequencerClient.scala index d52893483..e6c3e0ea0 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/sequencing/client/SequencerClient.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/sequencing/client/SequencerClient.scala @@ -330,33 +330,6 @@ class SequencerClient( SendAsyncClientError.DuplicateMessageId } - def performSend: EitherT[Future, SendAsyncClientError, Unit] = { - EitherTUtil - .timed(metrics.submissions.sends) { - val timeout = timeouts.network.duration - if (requiresAuthentication) - transport.sendAsync(request, timeout, staticDomainParameters.protocolVersion) - else - transport.sendAsyncUnauthenticated( - request, - timeout, - staticDomainParameters.protocolVersion, - ) - } - .leftSemiflatMap { err => - // increment appropriate error metrics - err match { - case SendAsyncClientError.RequestRefused(SendAsyncError.Overloaded(_)) => - metrics.submissions.overloaded.metric.inc() - case _ => - } - - // cancel pending send now as we know the request will never cause a sequenced result - logger.debug(s"Cancelling the pending send as the sequencer returned error: $err") - sendTracker.cancelPendingSend(messageId).map(_ => err) - } - } - if (replayEnabled) { EitherT.fromEither(for { _ <- unitOrBatchSizeErr @@ -380,11 +353,44 @@ class SequencerClient( _ <- EitherT.fromEither[Future](unitOrBatchSizeErr) _ <- trackSend _ = recorderO.foreach(_.recordSubmission(request)) - _ <- performSend + _ <- performSend(messageId, request, requiresAuthentication) } yield () } } + /** Perform the send, without any check. + */ + private def performSend( + messageId: MessageId, + request: SubmissionRequest, + requiresAuthentication: Boolean, + )(implicit traceContext: TraceContext): EitherT[Future, SendAsyncClientError, Unit] = { + EitherTUtil + .timed(metrics.submissions.sends) { + val timeout = timeouts.network.duration + if (requiresAuthentication) + transport.sendAsync(request, timeout, staticDomainParameters.protocolVersion) + else + transport.sendAsyncUnauthenticated( + request, + timeout, + staticDomainParameters.protocolVersion, + ) + } + .leftSemiflatMap { err => + // increment appropriate error metrics + err match { + case SendAsyncClientError.RequestRefused(SendAsyncError.Overloaded(_)) => + metrics.submissions.overloaded.metric.inc() + case _ => + } + + // cancel pending send now as we know the request will never cause a sequenced result + logger.debug(s"Cancelling the pending send as the sequencer returned error: $err") + sendTracker.cancelPendingSend(messageId).map(_ => err) + } + } + private def verifyBatchSize(batch: Batch[_]): Either[SendAsyncClientError, Unit] = { val batchSerializedSize = batch.toProtoVersioned(staticDomainParameters.protocolVersion).serializedSize @@ -1187,15 +1193,6 @@ object SequencerClient { connection, member, ) - // handshake to check that sequencer client supports the protocol version required by the sequencer - _ <- SequencerHandshake.handshake( - supportedProtocolVersions, - minimumProtocolVersion, - transport, - config, - processingTimeout, - loggerFactory, - ) // fetch the initial set of pending sends to initialize the client with. // as it owns the client that should be writing to this store it should not be racy. initialPendingSends <- EitherT.right(sendTrackerStore.fetchPendingSends) @@ -1252,32 +1249,46 @@ object SequencerClient { case grpc: GrpcSequencerConnection => grpcTransport(grpc, member).toEitherT } - replayConfigForMember(member) match { - case None => mkRealTransport - case Some(ReplayConfig(recording, SequencerEvents)) => - EitherT.rightT( - new ReplayingEventsSequencerClientTransport( + val transportEitherT: EitherT[Future, String, SequencerClientTransport] = + replayConfigForMember(member) match { + case None => mkRealTransport + case Some(ReplayConfig(recording, SequencerEvents)) => + EitherT.rightT( + new ReplayingEventsSequencerClientTransport( + domainParameters.protocolVersion, + recording.fullFilePath, + metrics, + processingTimeout, + loggerFactory, + ) + ) + case Some(ReplayConfig(recording, replaySendsConfig: SequencerSends)) => + for { + underlyingTransport <- mkRealTransport + } yield new ReplayingSendsSequencerClientTransport( domainParameters.protocolVersion, recording.fullFilePath, + replaySendsConfig, + member, + underlyingTransport, metrics, processingTimeout, loggerFactory, ) - ) - case Some(ReplayConfig(recording, replaySendsConfig: SequencerSends)) => - for { - underlyingTransport <- mkRealTransport - } yield new ReplayingSendsSequencerClientTransport( - domainParameters.protocolVersion, - recording.fullFilePath, - replaySendsConfig, - member, - underlyingTransport, - metrics, - processingTimeout, - loggerFactory, - ) - } + } + + for { + transport <- transportEitherT + // handshake to check that sequencer client supports the protocol version required by the sequencer + _ <- SequencerHandshake.handshake( + supportedProtocolVersions, + minimumProtocolVersion, + transport, + config, + processingTimeout, + loggerFactory, + ) + } yield transport } private def httpTransport(connection: HttpSequencerConnection)(implicit diff --git a/community/common/src/main/scala/com/digitalasset/canton/sequencing/client/transports/GrpcSequencerClientTransport.scala b/community/common/src/main/scala/com/digitalasset/canton/sequencing/client/transports/GrpcSequencerClientTransport.scala index 0b18f5570..8d1f5e120 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/sequencing/client/transports/GrpcSequencerClientTransport.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/sequencing/client/transports/GrpcSequencerClientTransport.scala @@ -56,12 +56,11 @@ class GrpcSequencerClientTransport( logger => traceContext => err match { - case _: GrpcClientGaveUp | _: GrpcServerError => + case _: GrpcClientGaveUp | _: GrpcServerError | _: GrpcServiceUnavailable => // avoid logging client errors that typically happen during shutdown (such as grpc context cancelled) - val _ = - performUnlessClosing("grpc-client-transport-log")(err.log(logger)(traceContext))( - traceContext - ) + performUnlessClosing("grpc-client-transport-log")(err.log(logger)(traceContext))( + traceContext + ).discard case _ => err.log(logger)(traceContext) } diff --git a/community/common/src/main/scala/com/digitalasset/canton/serialization/HasCryptographicEvidence.scala b/community/common/src/main/scala/com/digitalasset/canton/serialization/HasCryptographicEvidence.scala index ef8a7d36e..a3823a0f8 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/serialization/HasCryptographicEvidence.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/serialization/HasCryptographicEvidence.scala @@ -75,6 +75,8 @@ trait MemoizedEvidence extends HasCryptographicEvidence { /** Effectively immutable [[HasCryptographicEvidence]] classes can mix in this trait * to implement the memoization logic. + * Unlike [[MemoizedEvidence]] above, this trait should be mixed in classes that contain + * the protocol version. * * Use this class if serialization always succeeds. * @@ -84,7 +86,7 @@ trait MemoizedEvidence extends HasCryptographicEvidence { * * @see MemoizedEvidenceWithFailure if serialization may fail */ -trait MemoizedEvidenceV2 extends HasCryptographicEvidence { +trait ProtocolVersionedMemoizedEvidence extends HasCryptographicEvidence { /** Returns the [[com.google.protobuf.ByteString]] from which this object has been deserialized, if any. * If defined, [[getCryptographicEvidence]] will use this as the serialization. diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/LegalIdentityInit.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/LegalIdentityInit.scala index 59b74362a..a2d9ccb58 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/LegalIdentityInit.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/LegalIdentityInit.scala @@ -7,12 +7,15 @@ import cats.data.EitherT import cats.instances.future._ import cats.syntax.either._ import com.digitalasset.canton.crypto._ +import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.error.CantonError +import com.digitalasset.canton.topology.store.TopologyStore import com.digitalasset.canton.topology.transaction.{ + DomainTopologyTransactionType, LegalIdentityClaim, LegalIdentityClaimEvidence, - TopologyStateUpdate, SignedLegalIdentityClaim, + TopologyStateUpdate, } import com.digitalasset.canton.tracing.TraceContext @@ -22,7 +25,8 @@ class LegalIdentityInit(certificateGenerator: X509CertificateGenerator, crypto: ec: ExecutionContext, traceContext: TraceContext, ) { - def generateCertificate[E]( + + private def generateCertificate[E]( uid: UniqueIdentifier, alternativeNames: Seq[Member], ): EitherT[Future, String, X509Certificate] = @@ -46,39 +50,76 @@ class LegalIdentityInit(certificateGenerator: X509CertificateGenerator, crypto: .leftMap(err => s"Failed to store generated certificate: $err") } yield cert - def initializeCertificate[E <: CantonError]( + def getOrGenerateCertificate( uid: UniqueIdentifier, alternativeNames: Seq[Member], - namespaceKey: PublicKey, - )(topologyManager: TopologyManager[E]): EitherT[Future, String, Unit] = - for { - cert <- generateCertificate(uid, alternativeNames) + )(implicit traceContext: TraceContext): EitherT[Future, String, X509Certificate] = { - evidence <- cert.toPem - .map(LegalIdentityClaimEvidence.X509Cert) - .leftMap(err => s"Failed to serialize certificate to PEM: $err") - .toEitherT - claim = LegalIdentityClaim.create(uid, evidence) - claimHash = claim.hash(crypto.pureCrypto) + for { + certificateO <- crypto.cryptoPublicStore + .listCertificates() + .leftMap(err => s"Failed to list certificates in public crypto store: $err") + .map( + // TODO(soren) once we use proper uids for subject alternative names, we should include them in the filter + _.find(_.subjectCommonName.contains(uid.toProtoPrimitive)) + ) + certificate <- certificateO.fold(generateCertificate(uid, alternativeNames))(current => + EitherT.rightT(current) + ) + } yield certificate + } - // Sign the legal identity claim with the legal entity key as specified in the evidence - certKey <- cert - .publicKey(crypto.javaKeyConverter) - .leftMap(err => s"Failed to extract public key from certificate: $err") - .toEitherT - claimSig <- crypto.privateCrypto - .sign(claimHash, certKey.fingerprint) - .leftMap(err => s"Failed to sign legal identity claim: $err") + def checkOrInitializeCertificate[E <: CantonError]( + uid: UniqueIdentifier, + alternativeNames: Seq[Member], + namespaceKey: SigningPublicKey, + )(topologyManager: TopologyManager[E], store: TopologyStore): EitherT[Future, String, Unit] = + for { + cert <- getOrGenerateCertificate(uid, alternativeNames) - // Authorize the legal identity mapping with the namespace key - _ <- topologyManager - .authorize( - TopologyStateUpdate.createAdd( - SignedLegalIdentityClaim(uid, claim.getCryptographicEvidence, claimSig) - ), - Some(namespaceKey.fingerprint), - false, + // check store if there are existing transactions + current <- EitherT.right( + store.findPositiveTransactions( + asOf = CantonTimestamp.MaxValue, // max value will give us the "head state" + asOfInclusive = true, + includeSecondary = false, + types = Seq(DomainTopologyTransactionType.SignedLegalIdentityClaim), + filterUid = Some(Seq(uid)), + filterNamespace = None, ) - .leftMap(_.toString) + ) + + _ <- + if (current.adds.result.exists(_.transaction.key == namespaceKey)) + EitherT.rightT[Future, String](()) + else + for { + evidence <- cert.toPem + .map(LegalIdentityClaimEvidence.X509Cert) + .leftMap(err => s"Failed to serialize certificate to PEM: $err") + .toEitherT + claim = LegalIdentityClaim.create(uid, evidence) + claimHash = claim.hash(crypto.pureCrypto) + + // Sign the legal identity claim with the legal entity key as specified in the evidence + certKey <- cert + .publicKey(crypto.javaKeyConverter) + .leftMap(err => s"Failed to extract public key from certificate: $err") + .toEitherT + claimSig <- crypto.privateCrypto + .sign(claimHash, certKey.fingerprint) + .leftMap(err => s"Failed to sign legal identity claim: $err") + + // Authorize the legal identity mapping with the namespace key + _ <- topologyManager + .authorize( + TopologyStateUpdate.createAdd( + SignedLegalIdentityClaim(uid, claim.getCryptographicEvidence, claimSig) + ), + Some(namespaceKey.fingerprint), + false, + ) + .leftMap(_.toString) + } yield () } yield () } diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/Member.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/Member.scala index 8f988758b..5f6897376 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/Member.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/Member.scala @@ -8,7 +8,7 @@ import cats.syntax.either._ import com.daml.ledger.client.binding.Primitive.{Party => ClientParty} import com.digitalasset.canton.ProtoDeserializationError.ValueConversionError import com.digitalasset.canton.config.RequireTypes.{LengthLimitedString, String255, String300} -import com.digitalasset.canton.crypto.SecureRandomness +import com.digitalasset.canton.crypto.RandomOps import com.digitalasset.canton.logging.pretty.{Pretty, PrettyPrinting} import com.digitalasset.canton.serialization.ProtoConverter.ParsingResult import com.digitalasset.canton.store.db.DbDeserializationException @@ -190,10 +190,10 @@ object UnauthenticatedMemberId { private val RandomIdentifierNumberOfBytes = 20 - def tryCreate(namespace: Namespace): UnauthenticatedMemberId = + def tryCreate(namespace: Namespace)(randomOps: RandomOps): UnauthenticatedMemberId = UnauthenticatedMemberId( UniqueIdentifier.tryCreate( - HexString.toHexString(SecureRandomness.randomByteString(RandomIdentifierNumberOfBytes)), + HexString.toHexString(randomOps.generateRandomByteString(RandomIdentifierNumberOfBytes)), namespace.fingerprint.unwrap, ) ) diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/TopologyManager.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/TopologyManager.scala index d5bbe27a8..c3ec01cca 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/TopologyManager.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/TopologyManager.scala @@ -84,7 +84,7 @@ abstract class TopologyManager[E <: CantonError]( else { for { active <- EitherT.right( - store.findActiveTransactionsForMapping(transaction.transaction.element.mapping) + store.findPositiveTransactionsForMapping(transaction.transaction.element.mapping) ) filtered = active.find(sit => sit.transaction.element == transaction.transaction.element) _ <- EitherT.cond[Future]( @@ -148,20 +148,31 @@ abstract class TopologyManager[E <: CantonError]( } } - protected def checkMappingDoesNotExistYet( + def signedMappingAlreadyExists( + mapping: TopologyMapping, + signingKey: Fingerprint, + )(implicit traceContext: TraceContext): Future[Boolean] = + for { + txs <- store.findPositiveTransactionsForMapping(mapping) + mappings = txs.map(x => (x.transaction.element.mapping, x.key.fingerprint)) + } yield mappings.contains((mapping, signingKey)) + + protected def checkMappingOfTxDoesNotExistYet( transaction: SignedTopologyTransaction[TopologyChangeOp], allowDuplicateMappings: Boolean, )(implicit traceContext: TraceContext): EitherT[Future, TopologyManagerError, Unit] = - if (allowDuplicateMappings || transaction.transaction.op == TopologyChangeOp.Remove) { + if (allowDuplicateMappings || transaction.transaction.op != TopologyChangeOp.Add) { EitherT.rightT(()) } else { (for { - txs <- EitherT.right( - store.findActiveTransactionsForMapping(transaction.transaction.element.mapping) + exists <- EitherT.right( + signedMappingAlreadyExists( + transaction.transaction.element.mapping, + transaction.key.fingerprint, + ) ) - matching = txs.filter(SignedTopologyTransaction.equivalent(_, transaction)) _ <- EitherT.cond[Future]( - matching.isEmpty, + !exists, (), TopologyManagerError.MappingAlreadyExists.Failure( transaction.transaction.element, @@ -350,7 +361,7 @@ abstract class TopologyManager[E <: CantonError]( if (isUniquenessRequired) checkTransactionNotAddedBefore(transaction).leftMap(wrapError) else EitherT.pure[Future, E](()) _ <- checkRemovalRefersToExisingTx(transaction).leftMap(wrapError) - _ <- checkMappingDoesNotExistYet(transaction, allowDuplicateMappings).leftMap(wrapError) + _ <- checkMappingOfTxDoesNotExistYet(transaction, allowDuplicateMappings).leftMap(wrapError) _ <- transactionIsNotDangerous(transaction, force).leftMap(wrapError) _ <- checkNewTransaction(transaction, force) // domain / participant specific checks deactivateExisting <- removeExistingTransactions(transaction, replaceExisting) @@ -455,7 +466,7 @@ abstract class TopologyManager[E <: CantonError]( for { tx <- EitherT( store - .findActiveTransactionsForMapping(mapping) + .findPositiveTransactionsForMapping(mapping) .map( _.headOption.toRight[TopologyManagerError]( TopologyManagerError.NoCorrespondingActiveTxToRevoke.Mapping(mapping) diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/client/CachingDomainTopologyClient.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/client/CachingDomainTopologyClient.scala index 31567ef19..e23c9b54f 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/client/CachingDomainTopologyClient.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/client/CachingDomainTopologyClient.scala @@ -198,19 +198,13 @@ object CachingDomainTopologyClient { loggerFactory, ) - val initF = - // ts is "effective time", but during startup if we don't have an processed timestamp, we can use this one - store.timestamp(useStateStore = true).map { tsO => - tsO.map(ts => (ApproximateTime(ts), EffectiveTime(ts))) - } - - initF.map { initWithTsO => - initWithTsO.foreach { case (sequencedTs, effectiveTs) => - caching.updateHead(effectiveTs, sequencedTs, potentialTopologyChange = true) + store.timestamp(useStateStore = true).map { x => + x.foreach { case (sequenced, effective) => + caching + .updateHead(effective, ApproximateTime(sequenced.value), potentialTopologyChange = true) } caching } - } } diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/client/StoreBasedDomainTopologyClient.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/client/StoreBasedDomainTopologyClient.scala index e595b4acb..c1a878717 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/client/StoreBasedDomainTopologyClient.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/client/StoreBasedDomainTopologyClient.scala @@ -448,7 +448,7 @@ class StoreBasedTopologySnapshot( ), filterUid = Some(parties.map(_.uid)), filterNamespace = None, - ).map(_.toIdentityState) + ).map(_.toTopologyState) // aggregate the mappings, looking for matching request sides allAggregated = transactions.foldLeft(Map.empty[PartyId, PartyAggregation]) { @@ -498,7 +498,7 @@ class StoreBasedTopologySnapshot( types = Seq(DomainTopologyTransactionType.OwnerToKeyMapping), filterUid = Some(Seq(owner.uid)), filterNamespace = None, - ).map(_.toIdentityState) + ).map(_.toTopologyState) .map(_.collect { case TopologyStateUpdateElement(_, OwnerToKeyMapping(foundOwner, key)) if foundOwner.code == owner.code => @@ -522,7 +522,7 @@ class StoreBasedTopologySnapshot( types = Seq(DomainTopologyTransactionType.ParticipantState), filterUid = None, filterNamespace = None, - ).map(_.toIdentityState) + ).map(_.toTopologyState) // TODO(i4930) this is quite inefficient .map(_.collect { case TopologyStateUpdateElement(_, ps: ParticipantState) => ps.participant @@ -564,7 +564,7 @@ class StoreBasedTopologySnapshot( types = Seq(DomainTopologyTransactionType.ParticipantState), filterUid = Some(participants.map(_.uid)), filterNamespace = None, - ).map(_.toIdentityState) + ).map(_.toTopologyState) .map { loaded => loaded .foldLeft( @@ -611,7 +611,7 @@ class StoreBasedTopologySnapshot( types = Seq(DomainTopologyTransactionType.SignedLegalIdentityClaim), filterUid = Some(Seq(participantId.uid)), filterNamespace = None, - ).map(_.toIdentityState.reverse.collectFirstSome { + ).map(_.toTopologyState.reverse.collectFirstSome { case TopologyStateUpdateElement(_id, SignedLegalIdentityClaim(_, claimBytes, _signature)) => val result = for { claim <- LegalIdentityClaim @@ -651,7 +651,7 @@ class StoreBasedTopologySnapshot( namespaceOnly = false, ) .map { col => - col.toIdentityState + col.toTopologyState .map(_.mapping) .collect { case OwnerToKeyMapping(owner, key) @@ -693,7 +693,7 @@ class StoreBasedTopologySnapshot( filterUid = Some(Seq(participant.uid)), filterNamespace = None, ).map { res => - res.toIdentityState.flatMap { + res.toTopologyState.flatMap { case TopologyStateUpdateElement(_, VettedPackages(_, packageIds)) => packageIds case _ => Seq() }.toSet @@ -727,7 +727,7 @@ class StoreBasedTopologySnapshot( filterNamespace = None, ).map { res => ArraySeq.from( - res.toIdentityState + res.toTopologyState .foldLeft(Map.empty[MediatorId, (Boolean, Boolean)]) { case (acc, TopologyStateUpdateElement(_, MediatorDomainState(side, _, mediator))) => acc + (mediator -> RequestSide @@ -755,7 +755,7 @@ class StoreBasedTopologySnapshot( // We sort the results to be able to pick the most recent one in case // several transactions are found. val domainParameters = - StoredTopologyTransactions(storedTxs.result.sortBy(_.validFrom.value)).toIdentityState + StoredTopologyTransactions(storedTxs.result.sortBy(_.validFrom.value)).toTopologyState .collect { case DomainGovernanceElement(DomainParametersChange(_, domainParameters)) => domainParameters } diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/processing/TopologyTimestampPlusEpsilonTracker.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/processing/TopologyTimestampPlusEpsilonTracker.scala index b4ebfc146..1e3754a64 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/processing/TopologyTimestampPlusEpsilonTracker.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/processing/TopologyTimestampPlusEpsilonTracker.scala @@ -3,25 +3,29 @@ package com.digitalasset.canton.topology.processing -import cats.data.EitherT +import com.digitalasset.canton.DiscardOps +import com.digitalasset.canton.config.ProcessingTimeout import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.lifecycle.{FlagCloseable, FutureUnlessShutdown, UnlessShutdown} import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging} import com.digitalasset.canton.protocol.DynamicDomainParameters import com.digitalasset.canton.time._ -import com.digitalasset.canton.topology.client.StoreBasedTopologySnapshot import com.digitalasset.canton.topology.store.TopologyStore +import com.digitalasset.canton.topology.store.TopologyStore.Change +import com.digitalasset.canton.topology.transaction.{ + DomainParametersChange, + DomainTopologyTransactionType, +} import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.util.{ErrorUtil, FutureUtil} +import io.functionmeta.functionFullName +import java.util.ConcurrentModificationException import java.util.concurrent.atomic.AtomicReference import scala.annotation.tailrec -import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.concurrent.{ExecutionContext, Promise} import scala.util.{Failure, Success} -import com.digitalasset.canton.DiscardOps -import com.digitalasset.canton.config.ProcessingTimeout - -import java.util.ConcurrentModificationException +import com.daml.nonempty.NonEmpty /** Compute and synchronise the effective timestamps * @@ -146,7 +150,7 @@ class TopologyTimestampPlusEpsilonTracker( case None => FutureUnlessShutdown.pure(computeEffective) case Some(value) => logger.debug( - s"Need to wait until topology processing has caught up at $sequencingTime" + s"Need to wait until topology processing has caught up at $sequencingTime (must reach $synchronizeAt with current=${currentKnownTime})" ) value.map { _ => logger.debug(s"Topology processing caught up at $sequencingTime") @@ -232,103 +236,95 @@ class TopologyTimestampPlusEpsilonTracker( object TopologyTimestampPlusEpsilonTracker { - /** This is a new subscription (rather than a resubscription). - * Try to figure out whether this node was bootstrapped by checking whether we find something in the store. - * - * If so, approximate the sequencing time of the topology transactions from the - * [[com.digitalasset.canton.protocol.DynamicDomainParameters.WithValidity.validFrom]] - * and the new topology change delay. - */ - def initializeOnFirstSubscription( - tracker: TopologyTimestampPlusEpsilonTracker, + def epsilonForTimestamp( store: TopologyStore, + asOfExclusive: CantonTimestamp, )(implicit traceContext: TraceContext, executionContext: ExecutionContext, - ): FutureUnlessShutdown[EffectiveTime] = { - for { - // read the timestamp from the state store so we deal with crashes during startup / bootstrapping - // as we write first to the transaction store and then subsequently to the state store, - // but when we serve the topology state, then we serve it from the state store. - timestampO <- FutureUnlessShutdown.outcomeF(store.timestamp(useStateStore = true)) - topologyChangeDelay <- FutureUnlessShutdown.outcomeF( - determineEpsilonFromStore( - timestampO.getOrElse(CantonTimestamp.MinValue), - store, - tracker.loggerFactory, - ) - ) - effectiveTime <- adjustEpsilonAndTick( - tracker, - sequencedTs = timestampO.fold(CantonTimestamp.MinValue) { timestamp => - // Our best guess of when the topology change delay transaction was sequenced. - // Unfortunately, we cannot figure out when this change was really sequenced - // because the topology snapshot does not store the sequencing time of transactions. - // TODO(#1251) Include the original sequencing time of topology transactions during bootstrapping - timestamp - topologyChangeDelay - }, - topologyChangeDelay, - ) - } yield effectiveTime - } - - /** compute effective time of sequencedTs and populate t+e tracker with the current epsilon */ - def initializeFromStore( - tracker: TopologyTimestampPlusEpsilonTracker, - store: TopologyStore, - sequencedTs: CantonTimestamp, - )(implicit - traceContext: TraceContext, - executionContext: ExecutionContext, - ): FutureUnlessShutdown[EffectiveTime] = { + ): FutureUnlessShutdown[TopologyStore.Change.TopologyDelay] = { FutureUnlessShutdown - .outcomeF(determineEpsilonFromStore(sequencedTs, store, tracker.loggerFactory)) - .flatMap { topologyChangeDelay => - adjustEpsilonAndTick(tracker, sequencedTs, topologyChangeDelay) + .outcomeF( + store + .findPositiveTransactions( + asOf = asOfExclusive, + asOfInclusive = false, + includeSecondary = true, + types = Seq(DomainTopologyTransactionType.DomainParameters), + filterUid = None, + filterNamespace = None, + ) + ) + .map { txs => + txs.replaces.result + .map(x => (x.transaction.transaction.element.mapping, x)) + .collectFirst { case (change: DomainParametersChange, tx) => + TopologyStore.Change.TopologyDelay( + tx.sequenced, + tx.validFrom, + change.domainParameters.topologyChangeDelay, + ) + } + .getOrElse( + TopologyStore.Change.TopologyDelay( + SequencedTime(CantonTimestamp.MinValue), + EffectiveTime(CantonTimestamp.MinValue), + DynamicDomainParameters.topologyChangeDelayIfAbsent, + ) + ) } } - private def adjustEpsilonAndTick( + /** Initialize tracker + * + * @param processorTs Timestamp strictly (just) before the first message that will be passed: + * No sequenced events may have been passed in earlier crash epochs whose + * timestamp is strictly between `processorTs` and the first message that + * will be passed if these events affect the topology change delay. + * Normally, it's the timestamp of the last message that was successfully + * processed before the one that will be passed first. + */ + def initialize( tracker: TopologyTimestampPlusEpsilonTracker, - sequencedTs: CantonTimestamp, - topologyChangeDelay: NonNegativeFiniteDuration, - )(implicit - traceContext: TraceContext, - executionContext: ExecutionContext, - ): FutureUnlessShutdown[EffectiveTime] = { - // we initialize our tracker with the current topology change delay, using the sequenced timestamp - // as the effective time. this is fine as we know that if epsilon(sequencedTs) = e, then this e was activated by - // some transaction with effective time <= sequencedTs - tracker - .adjustEpsilon( - EffectiveTime(sequencedTs), - SequencedTime(CantonTimestamp.MinValue), - topologyChangeDelay, - ) - .discard[Option[NonNegativeFiniteDuration]] - tracker.adjustTimestampForTick(SequencedTime(sequencedTs)) - } - - private[topology] def determineEpsilonFromStore( - asOf: CantonTimestamp, store: TopologyStore, - loggerFactory: NamedLoggerFactory, + processorTs: CantonTimestamp, )(implicit traceContext: TraceContext, - ec: ExecutionContext, - ): Future[NonNegativeFiniteDuration] = { - val snapshot = - new StoreBasedTopologySnapshot( - asOf, - store, - Map(), - useStateTxs = true, - packageDependencies = _ => EitherT.pure(Set()), - loggerFactory, + executionContext: ExecutionContext, + ): FutureUnlessShutdown[EffectiveTime] = for { + // find the epsilon of a dpc asOf processorTs (which means it is exclusive) + epsilonAtProcessorTs <- epsilonForTimestamp(store, processorTs) + // find also all upcoming changes which have effective >= processorTs && sequenced <= processorTs + // the change that makes up the epsilon at processorTs would be grabbed by the statement above + upcoming <- tracker.performUnlessClosingF(functionFullName)( + store + .findUpcomingEffectiveChanges(processorTs) + .map(_.collect { + case tdc: Change.TopologyDelay + // filter anything out that might be replayed + if tdc.sequenced.value <= processorTs => + tdc + }) + ) + allPending = NonEmpty.mk(Seq, epsilonAtProcessorTs, upcoming: _*).sortBy(_.sequenced) + _ = { + tracker.logger.debug( + s"Initialising with $allPending" ) - snapshot - .findDynamicDomainParametersOrDefault(warnOnUsingDefault = false) - .map(_.topologyChangeDelay) - } + // Now, replay all the older epsilon updates that might get activated shortly + allPending.foreach { change => + tracker + .adjustEpsilon( + change.effective, + change.sequenced, + change.epsilon, + ) + .discard[Option[NonNegativeFiniteDuration]] + } + } + eff <- tracker.adjustTimestampForTick( + allPending.last1.sequenced + ) // need to init with the last one + } yield eff } diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/processing/TopologyTransactionProcessor.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/processing/TopologyTransactionProcessor.scala index f36c25e5c..80e799f15 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/processing/TopologyTransactionProcessor.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/processing/TopologyTransactionProcessor.scala @@ -20,8 +20,8 @@ import com.digitalasset.canton.protocol.messages.{ DomainTopologyTransactionMessage, ProtocolMessage, } -import com.digitalasset.canton.sequencing.protocol.{Batch, Deliver, DeliverError} import com.digitalasset.canton.sequencing._ +import com.digitalasset.canton.sequencing.protocol.{Batch, Deliver, DeliverError} import com.digitalasset.canton.serialization.ProtoConverter.ParsingResult import com.digitalasset.canton.time.Clock import com.digitalasset.canton.topology.client.{ @@ -47,7 +47,7 @@ import java.util.concurrent.atomic.AtomicBoolean import scala.collection.mutable.ListBuffer import scala.concurrent.{ExecutionContext, Future} -case class EffectiveTime(value: CantonTimestamp) extends AnyVal { +case class EffectiveTime(value: CantonTimestamp) { def toApproximate: ApproximateTime = ApproximateTime(value) def toProtoPrimitive: ProtoTimestamp = value.toProtoPrimitive @@ -62,13 +62,15 @@ object EffectiveTime { def fromProtoPrimitive(ts: ProtoTimestamp): ParsingResult[EffectiveTime] = CantonTimestamp.fromProtoPrimitive(ts).map(EffectiveTime(_)) } -case class ApproximateTime(value: CantonTimestamp) extends AnyVal -case class SequencedTime(value: CantonTimestamp) extends AnyVal { +case class ApproximateTime(value: CantonTimestamp) +case class SequencedTime(value: CantonTimestamp) { def toProtoPrimitive: ProtoTimestamp = value.toProtoPrimitive } object SequencedTime { def fromProtoPrimitive(ts: ProtoTimestamp): ParsingResult[SequencedTime] = CantonTimestamp.fromProtoPrimitive(ts).map(SequencedTime(_)) + implicit val orderingSequencedTime: Ordering[SequencedTime] = + Ordering.by[SequencedTime, CantonTimestamp](_.value) } trait TopologyTransactionProcessingSubscriber { @@ -136,63 +138,103 @@ class TopologyTransactionProcessor( private def initialise( start: ResubscriptionStart )(implicit traceContext: TraceContext): FutureUnlessShutdown[Unit] = { + ErrorUtil.requireState( !initialised.getAndSet(true), "topology processor is already initialised", ) - // TODO(#4638) initialize the topology processor and the topology client at the appropriate timestamps - val resubscriptionTs = start match { - case ResubscriptionStart.FreshSubscription => CantonTimestamp.MinValue - case ResubscriptionStart.ReplayResubscriptionStart(firstReplayed, cleanPreheadO) => - firstReplayed - case ResubscriptionStart.CleanHeadResubscriptionStart(cleanPrehead) => - cleanPrehead.immediateSuccessor - } + def resubscriptionTs( + timestamps: Option[(SequencedTime, EffectiveTime)] + ): (CantonTimestamp, Either[SequencedTime, EffectiveTime]) = + (start, timestamps) match { + // clean-head subscription. this means that the first event we are going to get is > cleanPrehead + // and all our stores are clean. + // processor: initialise with ts = cleanPrehead + // client: approximate time: cleanPrehead, knownUntil = cleanPrehead + epsilon + // plus, there might be "effective times" > cleanPrehead, so we need to schedule the adjustment + // of the approximate time to the effective time + case (ResubscriptionStart.CleanHeadResubscriptionStart(cleanPrehead), _) => + (cleanPrehead, Left(SequencedTime(cleanPrehead))) + // dirty or replay subscription. + // processor: initialise with firstReplayed.predecessor, as the next message we'll be getting is the firstReplayed + // client: same as clean-head resubscription + case ( + ResubscriptionStart.ReplayResubscriptionStart(firstReplayed, Some(cleanPrehead)), + _, + ) => + (firstReplayed.immediatePredecessor, Left(SequencedTime(cleanPrehead))) + // dirty re-subscription of a node that crashed before fully processing the first event + // processor: initialise with firstReplayed.predecessor, as the next message we'll be getting is the firstReplayed + // client: initialise client with firstReplayed (careful: firstReplayed is known, but firstReplayed.immediateSuccessor not) + case (ResubscriptionStart.ReplayResubscriptionStart(firstReplayed, None), _) => + ( + firstReplayed.immediatePredecessor, + Right(EffectiveTime(firstReplayed.immediatePredecessor)), + ) - def mergeEffectiveTimesWithApproximateTimestamps( - effectiveTsOfSequencedTs: EffectiveTime, - effectiveTimes: Seq[CantonTimestamp], - ): Seq[(EffectiveTime, ApproximateTime)] = { - (( - ( - effectiveTsOfSequencedTs, - ApproximateTime(resubscriptionTs), - ), - ) +: effectiveTimes.map(x => (EffectiveTime(x), ApproximateTime(x)))) - .sortBy(_._1.value) - } + // Fresh subscription with an empty domain topology store + // processor: init at ts = min + // client: init at ts = min + case (ResubscriptionStart.FreshSubscription, None) => + (CantonTimestamp.MinValue, Right(EffectiveTime(CantonTimestamp.MinValue))) + + // Fresh subscription with a bootstrapping timestamp + // NOTE: we assume that the bootstrapping topology snapshot does not contain the first message + // that we are going to receive from the domain + // processor: init at max(sequence-time) of bootstrapping transactions + // client: init at max(effective-time) of bootstrapping transactions + case (ResubscriptionStart.FreshSubscription, Some((sequenced, effective))) => + (sequenced.value, Right(effective)) - for { - // on startup, we need to figure out up to which point we know the topology state - effectiveKnownUntil <- start match { - case ResubscriptionStart.FreshSubscription => - TopologyTimestampPlusEpsilonTracker.initializeOnFirstSubscription( - timeAdjuster, - store, - ) - case _: ResubscriptionStart.ReplayResubscriptionStart | - _: ResubscriptionStart.CleanHeadResubscriptionStart => - TopologyTimestampPlusEpsilonTracker.initializeFromStore( - timeAdjuster, - store, - resubscriptionTs, - ) } + def initClientFromSequencedTs( + sequencedTs: SequencedTime + ): FutureUnlessShutdown[Seq[(EffectiveTime, ApproximateTime)]] = for { // we need to figure out any future effective time. if we had been running, there would be a clock // scheduled to poke the domain client at the given time in order to adjust the approximate timestamp up to the // effective time at the given point in time. we need to recover these as otherwise, we might be using outdated // topology snapshots on startup. (wouldn't be tragic as by getting the rejects, we'd be updating the timestamps // anyway). - futureEffectiveTimes <- performUnlessClosingF(functionFullName)( - store.findEffectiveTimestampsSince(resubscriptionTs) + upcoming <- performUnlessClosingF(functionFullName)( + store.findUpcomingEffectiveChanges(sequencedTs.value) + // find effective time of sequenced Ts (directly from store) + // merge times + ) + currentEpsilon <- TopologyTimestampPlusEpsilonTracker.epsilonForTimestamp( + store, + sequencedTs.value, ) } yield { - val tmp = - mergeEffectiveTimesWithApproximateTimestamps(effectiveKnownUntil, futureEffectiveTimes) - logger.debug(s"Initialising topology processing with effective ts ${tmp.map(_._1)}") - tmp.foreach { case (effective, approximate) => + + // we have (ts+e, ts) and quite a few te in the future, so we create list of upcoming changes and sort them + (( + EffectiveTime(sequencedTs.value.plus(currentEpsilon.epsilon.unwrap)), + ApproximateTime(sequencedTs.value), + ) +: upcoming.map(x => (x.effective, x.effective.toApproximate))).sortBy(_._1.value) + } + + for { + stateStoreTsO <- performUnlessClosingF(functionFullName)( + store.timestamp(useStateStore = true) + ) + (processorTs, clientTs) = resubscriptionTs(stateStoreTsO) + _ <- TopologyTimestampPlusEpsilonTracker.initialize(timeAdjuster, store, processorTs) + + clientInitTimes <- clientTs match { + case Left(sequencedTs) => + // approximate time is sequencedTs + initClientFromSequencedTs(sequencedTs) + case Right(effective) => + // effective and approximate time are effective time + FutureUnlessShutdown.pure(Seq((effective, effective.toApproximate))) + } + } yield { + logger.debug( + s"Initialising topology processing with effective ts ${clientInitTimes.map(_._1)}" + ) + clientInitTimes.foreach { case (effective, approximate) => // if the effective time is in the future, schedule a clock to update the time accordingly // TODO(ratko/andreas) This should be scheduled via the DomainTimeTracker or something similar // rather than via the participant's local clock diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/store/TopologyStore.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/store/TopologyStore.scala index eb2f8e2a8..b0fe1758c 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/store/TopologyStore.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/store/TopologyStore.scala @@ -21,6 +21,7 @@ import com.digitalasset.canton.topology.transaction._ import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.util.{ErrorUtil, MonadUtil} import com.digitalasset.canton.ProtoDeserializationError +import com.digitalasset.canton.time.NonNegativeFiniteDuration import com.digitalasset.canton.topology.processing.{EffectiveTime, SequencedTime} import com.google.common.annotations.VisibleForTesting @@ -206,15 +207,6 @@ object TimeQuery { for { fromO <- value.from.traverse(CantonTimestamp.fromProtoPrimitive) toO <- value.until.traverse(CantonTimestamp.fromProtoPrimitive) - _ <- Either.cond( - fromO.nonEmpty || toO.nonEmpty, - (), - ProtoDeserializationError - .ValueDeserializationError( - "range", - "At least one of from / to needs to be set within a time range query", - ), - ) } yield Range(fromO, toO) } @@ -269,7 +261,7 @@ abstract class TopologyStore(implicit ec: ExecutionContext) extends AutoCloseabl def timestamp(useStateStore: Boolean = false)(implicit traceContext: TraceContext - ): Future[Option[CantonTimestamp]] + ): Future[Option[(SequencedTime, EffectiveTime)]] /** set of topology transactions which are active */ def headTransactions(implicit @@ -283,9 +275,9 @@ abstract class TopologyStore(implicit ec: ExecutionContext) extends AutoCloseabl traceContext: TraceContext ): Future[Seq[SignedTopologyTransaction[TopologyChangeOp.Remove]]] - def findActiveTransactionsForMapping(mapping: TopologyMapping)(implicit + def findPositiveTransactionsForMapping(mapping: TopologyMapping)(implicit traceContext: TraceContext - ): Future[Seq[SignedTopologyTransaction[TopologyChangeOp.Add]]] + ): Future[Seq[SignedTopologyTransaction[TopologyChangeOp.Positive]]] @VisibleForTesting def allTransactions(implicit @@ -424,19 +416,48 @@ abstract class TopologyStore(implicit ec: ExecutionContext) extends AutoCloseabl filterNamespace: Option[Seq[Namespace]], )(implicit traceContext: TraceContext): Future[PositiveStoredTopologyTransactions] - /** fetch the effective time updates greater than a certain timestamp + /** fetch the effective time updates greater than or equal to a certain timestamp * * this function is used to recover the future effective timestamp such that we can reschedule "pokes" of the * topology client and updates of the acs commitment processor on startup */ - def findEffectiveTimestampsSince(timestamp: CantonTimestamp)(implicit + def findUpcomingEffectiveChanges(asOfInclusive: CantonTimestamp)(implicit traceContext: TraceContext - ): Future[Seq[CantonTimestamp]] + ): Future[Seq[TopologyStore.Change]] } object TopologyStore { + sealed trait Change extends Product with Serializable { + def sequenced: SequencedTime + def effective: EffectiveTime + } + + object Change { + case class TopologyDelay( + sequenced: SequencedTime, + effective: EffectiveTime, + epsilon: NonNegativeFiniteDuration, + ) extends Change + case class Other(sequenced: SequencedTime, effective: EffectiveTime) extends Change + + def accumulateUpcomingEffectiveChanges( + items: Seq[StoredTopologyTransaction[TopologyChangeOp]] + ): Seq[TopologyStore.Change] = { + items + .map(x => (x, x.transaction.transaction.element.mapping)) + .map { + case (tx, x: DomainParametersChange) => + TopologyDelay(tx.sequenced, tx.validFrom, x.domainParameters.topologyChangeDelay) + case (tx, _) => Other(tx.sequenced, tx.validFrom) + } + .sortBy(_.effective) + .distinct + } + + } + private[topology] case class InsertTransaction( transaction: SignedTopologyTransaction[TopologyChangeOp], validUntil: Option[CantonTimestamp], diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/store/TopologyTransactionCollection.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/store/TopologyTransactionCollection.scala index d98245f17..526575fbc 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/store/TopologyTransactionCollection.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/store/TopologyTransactionCollection.scala @@ -34,7 +34,7 @@ final case class StoredTopologyTransactions[+Op <: TopologyChangeOp]( _.result ) - def toIdentityState: List[TopologyStateElement[TopologyMapping]] = + def toTopologyState: List[TopologyStateElement[TopologyMapping]] = result.map(_.transaction.transaction.element).toList def toDomainTopologyTransactions: Seq[SignedTopologyTransaction[Op]] = @@ -200,7 +200,7 @@ final case class PositiveStoredTopologyTransactions( replaces: StoredTopologyTransactions[Replace], ) { def toIdentityState: List[TopologyStateElement[TopologyMapping]] = - adds.toIdentityState ++ replaces.toIdentityState + adds.toTopologyState ++ replaces.toTopologyState def combine: StoredTopologyTransactions[Positive] = StoredTopologyTransactions( adds.result ++ replaces.result diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/store/db/DbTopologyStore.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/store/db/DbTopologyStore.scala index f56c682e8..275b8eec2 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/store/db/DbTopologyStore.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/store/db/DbTopologyStore.scala @@ -432,7 +432,8 @@ class DbTopologyStore( )(implicit traceContext: TraceContext ): Future[CantonTimestamp] = - if (!isDomainStore) Future.successful(validFrom) // only compute for domain stores + if (!isDomainStore) + Future.successful(sequencedO.getOrElse(validFrom)) // only compute for domain stores else { def getParameterChangeBefore( ts: CantonTimestamp @@ -488,18 +489,10 @@ class DbTopologyStore( override def timestamp( useStateStore: Boolean - )(implicit traceContext: TraceContext): Future[Option[CantonTimestamp]] = { - + )(implicit traceContext: TraceContext): Future[Option[(SequencedTime, EffectiveTime)]] = { val storeId = if (useStateStore) stateStoreIdFilterName else transactionStoreIdName - val query = sql"""SELECT valid_from FROM topology_transactions - WHERE store_id = $storeId ORDER BY id DESC #${storage.limit(1)}""" - - readTime.metric - .event { - storage.query(query.as[Option[CantonTimestamp]].headOption, functionFullName) - } - .map(_.flatten) - + queryForTransactions(storeId, sql"", storage.limit(1), orderBy = " ORDER BY id DESC") + .map(_.result.headOption.map(tx => (tx.sequenced, tx.validFrom))) } override def headTransactions(implicit @@ -533,28 +526,26 @@ class DbTopologyStore( ) } - override def findActiveTransactionsForMapping( + override def findPositiveTransactionsForMapping( mapping: TopologyMapping )(implicit traceContext: TraceContext - ): Future[Seq[SignedTopologyTransaction[TopologyChangeOp.Add]]] = { + ): Future[Seq[SignedTopologyTransaction[TopologyChangeOp.Positive]]] = { val tmp = TopologyElementId.tryCreate("1") val ns = mapping.uniquePath(tmp).namespace val query = mapping.uniquePath(tmp).maybeUid.map(_.id) match { case None => sql"AND namespace = $ns" case Some(identifier) => sql"AND namespace = $ns AND identifier = $identifier" } - queryForTransactions( transactionStoreIdName, sql"AND valid_until is NULL AND transaction_type = ${mapping.dbType}" ++ query, ) - .map(_.result.collect { - case StoredTopologyTransaction(_sequenced, _validFrom, None, tr) - if tr.transaction.element.mapping == mapping => - tr - }) - .map(_.mapFilter(TopologyChangeOp.select[TopologyChangeOp.Add])) + .map { x => + x.positiveTransactions.combine.result + .map(_.transaction) + .filter(_.transaction.element.mapping == mapping) + } } override def allTransactions(implicit @@ -624,10 +615,10 @@ class DbTopologyStore( case None => sql" AND valid_until is NULL" } val query1: SQLActionBuilderChain = timeQuery match { - case TimeQuery.HeadState => getHeadStateQuery() + case TimeQuery.HeadState => + getHeadStateQuery() case TimeQuery.Snapshot(asOf) => asOfQuery(asOf = asOf, asOfInclusive = false) - case TimeQuery.Range(None, None) => sql"" // The case below insert an additional `AND` that we don't want case TimeQuery.Range(from, until) => @@ -868,21 +859,18 @@ class DbTopologyStore( }, ) - override def findEffectiveTimestampsSince( - timestamp: CantonTimestamp - )(implicit traceContext: TraceContext): Future[Seq[CantonTimestamp]] = { - val query = - sql"SELECT DISTINCT valid_from FROM topology_transactions WHERE store_id = $transactionStoreIdName AND valid_from > $timestamp ORDER BY valid_from" - readTime.metric.event { - storage - .query( - query.as[ - CantonTimestamp - ], - functionFullName, - ) - - } + override def findUpcomingEffectiveChanges(asOfInclusive: CantonTimestamp)(implicit + traceContext: TraceContext + ): Future[Seq[TopologyStore.Change]] = { + queryForTransactions( + transactionStoreIdName, + sql"AND valid_from >= $asOfInclusive ", + orderBy = " ORDER BY valid_from", + ).map(res => + TopologyStore.Change.accumulateUpcomingEffectiveChanges( + res.result + ) + ) } override def currentDispatchingWatermark(implicit diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/store/memory/InMemoryTopologyStore.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/store/memory/InMemoryTopologyStore.scala index 947c37a9e..1e224c733 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/store/memory/InMemoryTopologyStore.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/store/memory/InMemoryTopologyStore.scala @@ -197,10 +197,10 @@ class InMemoryTopologyStore(val loggerFactory: NamedLoggerFactory)(implicit ec: override def timestamp( useStateStore: Boolean - )(implicit traceContext: TraceContext): Future[Option[CantonTimestamp]] = + )(implicit traceContext: TraceContext): Future[Option[(SequencedTime, EffectiveTime)]] = Future.successful( - (if (useStateStore) topologyStateStore else topologyTransactionStore).lastOption.map( - _.from.value + (if (useStateStore) topologyStateStore else topologyTransactionStore).lastOption.map(x => + (x.sequenced, x.from) ) ) @@ -243,15 +243,15 @@ class InMemoryTopologyStore(val loggerFactory: NamedLoggerFactory)(implicit ec: } ) - override def findActiveTransactionsForMapping( + override def findPositiveTransactionsForMapping( mapping: TopologyMapping )(implicit traceContext: TraceContext - ): Future[Seq[SignedTopologyTransaction[Add]]] = + ): Future[Seq[SignedTopologyTransaction[Positive]]] = Future.successful( blocking(synchronized(topologyTransactionStore.toSeq)) .collect { case entry if entry.until.isEmpty => entry.transaction } - .mapFilter(TopologyChangeOp.select[Add]) + .mapFilter(TopologyChangeOp.select[Positive]) .collect { case sit if sit.transaction.element.mapping == mapping => sit } @@ -507,15 +507,15 @@ class InMemoryTopologyStore(val loggerFactory: NamedLoggerFactory)(implicit ec: ) } - override def findEffectiveTimestampsSince(timestamp: CantonTimestamp)(implicit + override def findUpcomingEffectiveChanges(asOfInclusive: CantonTimestamp)(implicit traceContext: TraceContext - ): Future[Seq[CantonTimestamp]] = + ): Future[Seq[TopologyStore.Change]] = Future.successful( - blocking(synchronized(topologyTransactionStore.toSeq)) - .map(_.from.value) - .filter(_ > timestamp) - .sorted - .distinct + TopologyStore.Change.accumulateUpcomingEffectiveChanges( + blocking(synchronized(topologyTransactionStore.toSeq)) + .filter(_.from.value >= asOfInclusive) + .map(_.toStoredTransaction) + ) ) private val watermark = new AtomicReference[Option[CantonTimestamp]](None) diff --git a/community/common/src/main/scala/com/digitalasset/canton/topology/transaction/SignedTopologyTransaction.scala b/community/common/src/main/scala/com/digitalasset/canton/topology/transaction/SignedTopologyTransaction.scala index f6c60eeca..3d452223a 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/topology/transaction/SignedTopologyTransaction.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/topology/transaction/SignedTopologyTransaction.scala @@ -8,7 +8,7 @@ import cats.syntax.either._ import com.digitalasset.canton.crypto._ import com.digitalasset.canton.logging.pretty.{Pretty, PrettyPrinting} import com.digitalasset.canton.protocol.v0 -import com.digitalasset.canton.serialization.{MemoizedEvidenceV2, ProtoConverter} +import com.digitalasset.canton.serialization.{ProtocolVersionedMemoizedEvidence, ProtoConverter} import com.digitalasset.canton.store.db.DbSerializationException import com.digitalasset.canton.version.{ HasMemoizedProtocolVersionedWrapperCompanion, @@ -41,7 +41,7 @@ case class SignedTopologyTransaction[+Op <: TopologyChangeOp]( val deserializedFrom: Option[ByteString] = None, ) extends HasProtocolVersionedWrapper[VersionedMessage[SignedTopologyTransaction[TopologyChangeOp]]] with HasProtoV0[v0.SignedTopologyTransaction] - with MemoizedEvidenceV2 + with ProtocolVersionedMemoizedEvidence with Product with Serializable with PrettyPrinting { @@ -51,7 +51,7 @@ case class SignedTopologyTransaction[+Op <: TopologyChangeOp]( override protected def toProtoVersioned : VersionedMessage[SignedTopologyTransaction[TopologyChangeOp]] = - SignedTopologyTransaction.toProtoVersionedV2(this) + SignedTopologyTransaction.toProtoVersioned(this) override protected def toProtoV0: v0.SignedTopologyTransaction = v0.SignedTopologyTransaction( diff --git a/community/common/src/main/scala/com/digitalasset/canton/version/HasProtocolVersionedWrapper.scala b/community/common/src/main/scala/com/digitalasset/canton/version/HasProtocolVersionedWrapper.scala index 0a8159eab..feb513b8b 100644 --- a/community/common/src/main/scala/com/digitalasset/canton/version/HasProtocolVersionedWrapper.scala +++ b/community/common/src/main/scala/com/digitalasset/canton/version/HasProtocolVersionedWrapper.scala @@ -3,13 +3,13 @@ package com.digitalasset.canton.version -import com.daml.nonempty.NonEmpty -import com.digitalasset.canton.ProtoDeserializationError +import com.daml.nonempty.{NonEmpty, NonEmptyUtil} +import com.digitalasset.canton.{ProtoDeserializationError, checked} import com.digitalasset.canton.serialization.ProtoConverter import com.digitalasset.canton.serialization.ProtoConverter.ParsingResult import com.google.protobuf.ByteString -import scala.collection.SortedMap +import scala.collection.immutable /** Trait for classes that can be serialized by using ProtoBuf. * See "CONTRIBUTING.md" for our guidelines on serialization. @@ -55,7 +55,7 @@ trait HasProtocolVersionedWrapper[+ProtoClass <: scalapb.GeneratedMessage] { def getCryptographicEvidence: ByteString } -trait HasMemoizedProtocolVersionedWrapperCompanion[ValueClass <: HasProtocolVersionedWrapper[ +trait GenericHasMemoizedProtocolVersionedWrapperCompanion[ValueClass <: HasProtocolVersionedWrapper[ VersionedMessage[ValueClass] ]] { @@ -66,10 +66,16 @@ trait HasMemoizedProtocolVersionedWrapperCompanion[ValueClass <: HasProtocolVers type DataByteString = ByteString // What is inside the parsed UntypedVersionedMessage message // Deserializer: (Proto => ValueClass) - type Deserializer = (OriginalByteString, DataByteString) => ParsingResult[ValueClass] + type Deserializer // Serializer: (ValueClass => Proto) type Serializer = ValueClass => ByteString + def protocolVersionRepresentativeFor(protocolVersion: ProtocolVersion): ProtocolVersion = + supportedProtoVersions.protocolVersionRepresentativeFor(protocolVersion) + + def protocolVersionRepresentativeFor(protoVersion: Int): ProtocolVersion = + supportedProtoVersions.protocolVersionRepresentativeFor(protoVersion) + /** Supported protobuf version * @param fromInclusive The protocol version when this protobuf version was introduced * @param deserializer Deserialization method @@ -81,15 +87,11 @@ trait HasMemoizedProtocolVersionedWrapperCompanion[ValueClass <: HasProtocolVers serializer: Serializer, ) - @SuppressWarnings(Array("org.wartremover.warts.TraversableOps")) sealed abstract case class SupportedProtoVersions( - converters: SortedMap[Int, VersionedProtoConverter] // Sorted with descending order + // Sorted with descending order + converters: NonEmpty[immutable.SortedMap[Int, VersionedProtoConverter]] ) { - require( - converters.nonEmpty, - "List of converters should not be empty", - ) // Prevented by factory method below - val (higherProtoVersion, higherConverter) = converters.head + val (higherProtoVersion, higherConverter) = converters.head1 def deserializerFor(protoVersion: Int): Deserializer = converters.get(protoVersion).map(_.deserializer).getOrElse(higherConverter.deserializer) @@ -101,7 +103,7 @@ trait HasMemoizedProtocolVersionedWrapperCompanion[ValueClass <: HasProtocolVers .getOrElse(higherConverter.fromInclusive) def protocolVersionRepresentativeFor(protocolVersion: ProtocolVersion): ProtocolVersion = - supportedProtoVersions.converters + converters .collectFirst { case (_, supportedVersion) if protocolVersion >= supportedVersion.fromInclusive => supportedVersion.fromInclusive @@ -114,19 +116,21 @@ trait HasMemoizedProtocolVersionedWrapperCompanion[ValueClass <: HasProtocolVers head: (Int, VersionedProtoConverter), tail: (Int, VersionedProtoConverter)* ): SupportedProtoVersions = SupportedProtoVersions( - NonEmpty.mk(Seq, head, tail: _*).toMap + NonEmpty.mk(Seq, head, tail: _*) ) - @SuppressWarnings(Array("org.wartremover.warts.TraversableOps")) - def apply(converters: NonEmpty[Map[Int, VersionedProtoConverter]]): SupportedProtoVersions = { - - val sortedConverters = SortedMap.from(converters)(implicitly[Ordering[Int]].reverse) - val (_, lowestProtocolVersion) = sortedConverters.last - - if (lowestProtocolVersion.fromInclusive != ProtocolVersion.minimum_protocol_version) - throw new IllegalArgumentException( - s"ProtocolVersion corresponding to lowest proto version should be ${ProtocolVersion.minimum_protocol_version}, found $lowestProtocolVersion" + def apply(converters: NonEmpty[Seq[(Int, VersionedProtoConverter)]]): SupportedProtoVersions = { + val sortedConverters = checked( + NonEmptyUtil.fromUnsafe( + immutable.SortedMap.from(converters)(implicitly[Ordering[Int]].reverse) ) + ) + val (_, lowestProtocolVersion) = sortedConverters.last1 + + require( + lowestProtocolVersion.fromInclusive == ProtocolVersion.minimum_protocol_version, + s"ProtocolVersion corresponding to lowest proto version should be ${ProtocolVersion.minimum_protocol_version}, found $lowestProtocolVersion", + ) new SupportedProtoVersions(sortedConverters) {} } @@ -137,15 +141,7 @@ trait HasMemoizedProtocolVersionedWrapperCompanion[ValueClass <: HasProtocolVers */ def supportedProtoVersions: SupportedProtoVersions - protected def supportedProtoVersionMemoized[Proto <: scalapb.GeneratedMessage]( - p: scalapb.GeneratedMessageCompanion[Proto] - )( - fromProto: Proto => (OriginalByteString => ParsingResult[ValueClass]) - ): Deserializer = - (original: OriginalByteString, data: DataByteString) => - ProtoConverter.protoParser(p.parseFrom)(data).flatMap(fromProto(_)(original)) - - def toProtoVersionedV2(v: ValueClass): VersionedMessage[ValueClass] = + def toProtoVersioned(v: ValueClass): VersionedMessage[ValueClass] = supportedProtoVersions.converters .collectFirst { case (protoVersion, supportedVersion) @@ -158,6 +154,21 @@ trait HasMemoizedProtocolVersionedWrapperCompanion[ValueClass <: HasProtocolVers supportedProtoVersions.higherProtoVersion, ) } +} + +trait HasMemoizedProtocolVersionedWrapperCompanion[ValueClass <: HasProtocolVersionedWrapper[ + VersionedMessage[ValueClass] +]] extends GenericHasMemoizedProtocolVersionedWrapperCompanion[ValueClass] { + // Deserializer: (Proto => ValueClass) + type Deserializer = (OriginalByteString, DataByteString) => ParsingResult[ValueClass] + + protected def supportedProtoVersionMemoized[Proto <: scalapb.GeneratedMessage]( + p: scalapb.GeneratedMessageCompanion[Proto] + )( + fromProto: Proto => (OriginalByteString => ParsingResult[ValueClass]) + ): Deserializer = + (original: OriginalByteString, data: DataByteString) => + ProtoConverter.protoParser(p.parseFrom)(data).flatMap(fromProto(_)(original)) def fromByteString(bytes: OriginalByteString): ParsingResult[ValueClass] = for { proto <- ProtoConverter.protoParser(UntypedVersionedMessage.parseFrom)(bytes) @@ -165,3 +176,24 @@ trait HasMemoizedProtocolVersionedWrapperCompanion[ValueClass <: HasProtocolVers valueClass <- supportedProtoVersions.deserializerFor(proto.version)(bytes, data) } yield valueClass } + +trait HasMemoizedProtocolVersionedWithContextCompanion[ValueClass <: HasProtocolVersionedWrapper[ + VersionedMessage[ValueClass] +], Context] + extends GenericHasMemoizedProtocolVersionedWrapperCompanion[ValueClass] { + type Deserializer = (Context, OriginalByteString, DataByteString) => ParsingResult[ValueClass] + + protected def supportedProtoVersionMemoized[Proto <: scalapb.GeneratedMessage]( + p: scalapb.GeneratedMessageCompanion[Proto] + )( + fromProto: (Context, Proto) => (OriginalByteString => ParsingResult[ValueClass]) + ): Deserializer = + (ctx: Context, original: OriginalByteString, data: DataByteString) => + ProtoConverter.protoParser(p.parseFrom)(data).flatMap(fromProto(ctx, _)(original)) + + def fromByteString(context: Context)(bytes: OriginalByteString): ParsingResult[ValueClass] = for { + proto <- ProtoConverter.protoParser(UntypedVersionedMessage.parseFrom)(bytes) + data <- proto.wrapper.data.toRight(ProtoDeserializationError.FieldNotSet(s"$name: data")) + valueClass <- supportedProtoVersions.deserializerFor(proto.version)(context, bytes, data) + } yield valueClass +} diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/EncryptionTest.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/EncryptionTest.scala index 10c47bbd7..266674f0b 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/EncryptionTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/EncryptionTest.scala @@ -38,8 +38,8 @@ trait EncryptionTest extends BaseTest { this: AsyncWordSpec => .generateSymmetricKey(scheme = symmetricKeyScheme) .valueOrFail("generate symmetric key") - def newSecureRandomKey(): SecureRandomness = { - SecureRandomness.secureRandomness(symmetricKeyScheme.keySizeInBytes) + def newSecureRandomKey(crypto: Crypto): SecureRandomness = { + crypto.pureCrypto.generateSecureRandomness(symmetricKeyScheme.keySizeInBytes) } "serialize and deserialize symmetric encryption key via protobuf" in { @@ -85,7 +85,7 @@ trait EncryptionTest extends BaseTest { this: AsyncWordSpec => for { crypto <- newCrypto message = Message(ByteString.copyFromUtf8("foobar")) - key = newSecureRandomKey() + key = newSecureRandomKey(crypto) encrypted = crypto.pureCrypto .encryptWith(message, key, ProtocolVersion.latestForTest) .valueOrFail("encrypt") @@ -102,8 +102,8 @@ trait EncryptionTest extends BaseTest { this: AsyncWordSpec => for { crypto <- newCrypto message = Message(ByteString.copyFromUtf8("foobar")) - key = newSecureRandomKey() - key2 = newSecureRandomKey() + key = newSecureRandomKey(crypto) + key2 = newSecureRandomKey(crypto) encrypted = crypto.pureCrypto .encryptWith(message, key, ProtocolVersion.latestForTest) .valueOrFail("encrypt") diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/HkdfTest.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/HkdfTest.scala index 6b820cc76..2a671387e 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/HkdfTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/HkdfTest.scala @@ -13,17 +13,103 @@ import scala.concurrent.Future trait HkdfTest { this: AsyncWordSpec with BaseTest => - def hkdfProvider(hkdfF: => Future[HkdfOps]): Unit = { + private case class TestCase( + ikm: ByteString, + salt: ByteString, + info: ByteString, + length: Int, + prk: ByteString, + okm: ByteString, + ) + + private object TestCase { + def apply( + ikmS: String, + saltS: String, + infoS: String, + length: Int, + prkS: String, + okmS: String, + ): TestCase = { + def parse(input: String, desc: String): ByteString = { + HexString + .parseToByteString(input.stripMargin.filter(_ != '\n')) + .valueOrFail(s"Invalid $desc string: $input") + } + + val ikm = parse(ikmS, "input key material") + val salt = parse(saltS, "salt") + val info = parse(infoS, "info") + val prk = parse(prkS, "pseudo-random key") + val okm = parse(okmS, "output key material") + + TestCase(ikm = ikm, salt = salt, info = info, length = length, prk = prk, okm = okm) + } + } + + private lazy val testCases = List( + TestCase( + ikmS = "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + saltS = "000102030405060708090a0b0c", + infoS = "f0f1f2f3f4f5f6f7f8f9", + length = 42, + prkS = """077709362c2e32df0ddc3f0dc47bba63 + |90b6c73bb50f9c3122ec844ad7c2b3e5""", + okmS = """3cb25f25faacd57a90434f64d0362f2a + |2d2d0a90cf1a5a4c5db02d56ecc4c5bf + |34007208d5b887185865""", + ), + TestCase( + ikmS = """000102030405060708090a0b0c0d0e0f + |101112131415161718191a1b1c1d1e1f + |202122232425262728292a2b2c2d2e2f + |303132333435363738393a3b3c3d3e3f + |404142434445464748494a4b4c4d4e4f""", + saltS = """606162636465666768696a6b6c6d6e6f + |707172737475767778797a7b7c7d7e7f + |808182838485868788898a8b8c8d8e8f + |909192939495969798999a9b9c9d9e9f + |a0a1a2a3a4a5a6a7a8a9aaabacadaeaf""", + infoS = """b0b1b2b3b4b5b6b7b8b9babbbcbdbebf + |c0c1c2c3c4c5c6c7c8c9cacbcccdcecf + |d0d1d2d3d4d5d6d7d8d9dadbdcdddedf + |e0e1e2e3e4e5e6e7e8e9eaebecedeeef + |f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff""", + length = 82, + prkS = """06a6b88c5853361a06104c9ceb35b45c + |ef760014904671014a193f40c15fc244""", + okmS = """b11e398dc80327a1c8e7f78c596a4934 + |4f012eda2d4efad8a050cc4c19afa97c + |59045a99cac7827271cb41c65e590e09 + |da3275600c2f09b8367793a9aca3db71 + |cc30c58179ec3e87c14c01d5c1f3434f + |1d87""", + ), + TestCase( + ikmS = "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + saltS = "", + infoS = "", + length = 42, + prkS = """19ef24a32c717b167f33a91d6f648bdf + |96596776afdb6377ac434c1c293ccb04""", + okmS = """8da4e775a563c18f715f802a063c5a31 + |b8a11f5c5ee1879ec3454e5f3c738d2d + |9d201395faa4b61a96c8""", + ), + ) + + def hkdfProvider(providerF: => Future[HkdfOps with RandomOps]): Unit = { "HKDF provider" should { "produce an output of the specified length" in { val algo = HmacAlgorithm.HmacSha256 val algoLen = algo.hashAlgorithm.length - val secret = SecureRandomness.secureRandomness(algoLen.toInt) - hkdfF.map { hkdf => + providerF.map { provider => + val secret = provider.generateSecureRandomness(algoLen.toInt) + // Test a few key sizes that we might need forAll(0L until (5L * algoLen)) { i => val expanded = - hkdf + provider .hkdfExpand( secret, i.toInt, @@ -36,51 +122,39 @@ trait HkdfTest { } } - "pass golden tests from RFC 5869" in { - val vectors = List( - ( - "077709362c2e32df0ddc3f0dc47bba6390b6c73bb50f9c3122ec844ad7c2b3e5", - "f0f1f2f3f4f5f6f7f8f9", - 42, - "3cb25f25faacd57a90434f64d0362f2a2d2d0a90cf1a5a4c5db02d56ecc4c5bf34007208d5b887185865", - ), - ( - "06a6b88c5853361a06104c9ceb35b45cef760014904671014a193f40c15fc244", - """b0b1b2b3b4b5b6b7b8b9babbbcbdbebf - |c0c1c2c3c4c5c6c7c8c9cacbcccdcecf - |d0d1d2d3d4d5d6d7d8d9dadbdcdddedf - |e0e1e2e3e4e5e6e7e8e9eaebecedeeef - |f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff""".stripMargin.filter(_ != '\n'), - 82, - """b11e398dc80327a1c8e7f78c596a4934 - |4f012eda2d4efad8a050cc4c19afa97c - |59045a99cac7827271cb41c65e590e09 - |da3275600c2f09b8367793a9aca3db71 - |cc30c58179ec3e87c14c01d5c1f3434f - |1d87""".stripMargin.filter(_ != '\n'), - ), - ( - "19ef24a32c717b167f33a91d6f648bdf96596776afdb6377ac434c1c293ccb04", - "", - 42, - "8da4e775a563c18f715f802a063c5a31b8a11f5c5ee1879ec3454e5f3c738d2d9d201395faa4b61a96c8", - ), - ) + "pass golden tests from RFC 5869 for extract-and-expand" in { val algo = HmacAlgorithm.HmacSha256 - hkdfF.map { hkdf => - forAll(vectors) { case (keyString, infoString, len, expectedOut) => - val key = HexString - .parseToByteString(keyString) - .getOrElse(throw new IllegalStateException(s"Wrong test vector key bytes $keyString")) - val info = HexString - .parseToByteString(infoString) - .getOrElse(throw new IllegalStateException(s"Wrong test vector info $infoString")) + providerF.map { provider => + forAll(testCases) { testCase => val expanded = - hkdf - .hkdfExpand(SecureRandomness(key), len, HkdfInfo.testOnly(info), algo) + provider + .computeHkdf( + testCase.ikm, + testCase.length, + HkdfInfo.testOnly(testCase.info), + testCase.salt, + algo, + ) + .valueOrFail("Could not compute the HMAC for test vector") + expanded.unwrap shouldBe testCase.okm + } + } + } + + "pass golden tests from RFC 5869 for expand" in { + val algo = HmacAlgorithm.HmacSha256 + providerF.map { provider => + forAll(testCases) { testCase => + val expanded = + provider + .hkdfExpand( + SecureRandomness(testCase.prk), + testCase.length, + HkdfInfo.testOnly(testCase.info), + algo, + ) .valueOrFail("Could not compute the HMAC for test vector") - val expandedHex = HexString.toHexString(expanded.unwrap) - expandedHex shouldBe expectedOut + expanded.unwrap shouldBe testCase.okm } } } diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/HmacTest.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/HmacTest.scala index 8e82f01f2..585623e51 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/HmacTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/HmacTest.scala @@ -5,20 +5,20 @@ package com.digitalasset.canton.crypto import cats.syntax.either._ import com.digitalasset.canton.BaseTest +import com.digitalasset.canton.crypto.provider.symbolic.SymbolicPureCrypto import com.google.protobuf.ByteString -import org.scalatest.wordspec.{AnyWordSpec, AsyncWordSpec} - -import scala.concurrent.Future +import org.scalatest.wordspec.AnyWordSpec class HmacTest extends AnyWordSpec with BaseTest { - private val longString = PseudoRandom.randomAlphaNumericString(256) + private lazy val longString = PseudoRandom.randomAlphaNumericString(256) + private lazy val crypto = new SymbolicPureCrypto forAll(HmacAlgorithm.algorithms) { algorithm => s"HMAC ${algorithm.name}" should { "serializing and deserializing via protobuf" in { - val secret = HmacSecret.generate() + val secret = HmacSecret.generate(crypto) val hmac = Hmac .compute(secret, ByteString.copyFromUtf8(longString), algorithm) @@ -32,39 +32,3 @@ class HmacTest extends AnyWordSpec with BaseTest { } } - -trait HmacPrivateTest extends BaseTest { this: AsyncWordSpec => - - def hmacProvider(newHmacPrivateOps: => Future[HmacPrivateOps]): Unit = { - - "HMAC private ops" should { - - "initialize and compute an HMAC with a stored secret" in { - for { - privateCrypto <- newHmacPrivateOps - _ <- privateCrypto.initializeHmacSecret().valueOrFail("init hmac secret") - _ <- privateCrypto.hmac(ByteString.copyFromUtf8("foobar")) - } yield assert(true) - } - - "rotate an HMAC secret" in { - for { - privateCrypto <- newHmacPrivateOps - data = ByteString.copyFromUtf8("foobar") - _ <- privateCrypto.initializeHmacSecret().valueOrFail("init hmac secret") - hmac1 <- privateCrypto.hmac(data).valueOrFail("create hmac1") - hmac2 <- privateCrypto.hmac(data).valueOrFail("create hmac2") - _ <- privateCrypto.rotateHmacSecret().valueOrFail("rotate hmac secret") - hmac3 <- privateCrypto.hmac(data).valueOrFail("create hmac3") - } yield { - hmac1 shouldEqual hmac2 - hmac1 should not equal hmac3 - } - - } - - } - - } - -} diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/RandomTest.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/RandomTest.scala new file mode 100644 index 000000000..735d2b1e6 --- /dev/null +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/RandomTest.scala @@ -0,0 +1,27 @@ +// Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.digitalasset.canton.crypto + +import com.digitalasset.canton.BaseTest +import org.scalatest.wordspec.AsyncWordSpec + +import scala.concurrent.Future + +trait RandomTest { + this: AsyncWordSpec with BaseTest => + + def randomnessProvider(providerF: => Future[RandomOps]): Unit = { + "Randomness provider" should { + "generate fresh randomness" in { + + providerF.map { provider => + val random1 = provider.generateRandomByteString(32) + val random2 = provider.generateRandomByteString(32) + + random1 should not equal random2 + } + } + } + } +} diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/SaltTest.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/SaltTest.scala index c26f842c1..7969bf1d6 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/SaltTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/SaltTest.scala @@ -4,37 +4,31 @@ package com.digitalasset.canton.crypto import com.digitalasset.canton.BaseTest -import com.digitalasset.canton.concurrent.DirectExecutionContext import com.digitalasset.canton.crypto.provider.symbolic.{SymbolicCrypto, SymbolicPureCrypto} -import com.google.protobuf.ByteString import org.scalatest.wordspec.AnyWordSpec -import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContext} - class SaltTest extends AnyWordSpec with BaseTest { "Salt" should { "serializing and deserializing via protobuf" in { - val salt = TestSalt.generate(0) + val salt = TestSalt.generateSalt(0) val saltP = salt.toProtoV0 Salt.fromProtoV0(saltP).value shouldBe salt } - "generate a fresh salt" in { - implicit val ec: ExecutionContext = DirectExecutionContext(logger) - + "generate a fresh salt seeds" in { val crypto = SymbolicCrypto.create(timeouts, loggerFactory) - val seedData = ByteString.copyFromUtf8("testSeedData") - val salt = Await.result(Salt.generate(seedData, crypto.privateCrypto).value, 10.seconds) + val salt1 = SaltSeed.generate()(crypto.pureCrypto) + val salt2 = SaltSeed.generate()(crypto.pureCrypto) - salt.value shouldBe a[Salt] + salt1 shouldBe a[SaltSeed] + salt1 should not equal salt2 } "derive a salt" in { val hmacOps = new SymbolicPureCrypto - val seedSalt = TestSalt.generate(0) + val seedSalt = TestSalt.generateSeed(0) val salt = Salt.deriveSalt(seedSalt, 0, hmacOps) // The derived salt must be different than the seed salt value diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/TestHkdf.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/TestHkdf.scala deleted file mode 100644 index 810520f22..000000000 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/TestHkdf.scala +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.digitalasset.canton.crypto - -/** A testing hkdf that simply concatenates the given info to the given key material, ignoring the length argument. - * Suitable for use with symbolic crypto only. - */ -class TestHkdf extends HkdfOps with HmacOps { - override def hkdfExpand( - keyMaterial: SecureRandomness, - outputBytes: Int, - info: HkdfInfo, - algorithm: HmacAlgorithm, - ): Either[HkdfError, SecureRandomness] = { - Right(SecureRandomness(keyMaterial.unwrap.concat(info.bytes))) - } -} diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/TestHmacSecret.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/TestHmacSecret.scala deleted file mode 100644 index 51794e2a9..000000000 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/TestHmacSecret.scala +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.digitalasset.canton.crypto - -import cats.syntax.either._ -import com.google.protobuf.ByteString - -object TestHmacSecret { - - def generate(): HmacSecret = { - val pseudoSecret = PseudoRandom.randomAlphaNumericString(HmacSecret.defaultLength) - HmacSecret - .create(ByteString.copyFromUtf8(pseudoSecret)) - .valueOr(err => throw new IllegalStateException(err.toString)) - } - -} diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/TestSalt.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/TestSalt.scala index 9aaaeeda6..4f97ba43b 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/TestSalt.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/TestSalt.scala @@ -7,9 +7,13 @@ import cats.syntax.either._ object TestSalt { + def generateSeed(index: Int): SaltSeed = { + SaltSeed(TestHash.digest(index).unwrap) + } + // Generates a deterministic salt for hashing based on the provided index // Assumes TestHash uses SHA-256 - def generate(index: Int): Salt = + def generateSalt(index: Int): Salt = Salt .create(TestHash.digest(index).unwrap, SaltAlgorithm.Hmac(HmacAlgorithm.HmacSha256)) .valueOr(err => throw new IllegalStateException(err.toString)) diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/jce/JceCryptoTest.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/jce/JceCryptoTest.scala index 7b03aede7..888f64f13 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/jce/JceCryptoTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/jce/JceCryptoTest.scala @@ -15,8 +15,8 @@ class JceCryptoTest extends AsyncWordSpec with SigningTest with EncryptionTest - with HmacPrivateTest with HkdfTest + with RandomTest with JavaKeyConverterTest { "JceCrypto" can { @@ -33,8 +33,8 @@ class JceCryptoTest behave like signingProvider(Jce.signing.supported, jceCrypto()) behave like encryptionProvider(Jce.encryption.supported, Jce.symmetric.supported, jceCrypto()) - behave like hmacProvider(jceCrypto().map(_.privateCrypto)) behave like hkdfProvider(jceCrypto().map(_.pureCrypto)) + behave like randomnessProvider(jceCrypto().map(_.pureCrypto)) behave like javaKeyConverterProvider( Jce.signing.supported, Jce.encryption.supported, diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicCrypto.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicCrypto.scala index 846d2d60e..1fc6b596c 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicCrypto.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicCrypto.scala @@ -3,7 +3,6 @@ package com.digitalasset.canton.crypto.provider.symbolic -import java.security.{PrivateKey => JPrivateKey, PublicKey => JPublicKey} import com.digitalasset.canton.concurrent.DirectExecutionContext import com.digitalasset.canton.config.ProcessingTimeout import com.digitalasset.canton.crypto._ @@ -12,13 +11,12 @@ import com.digitalasset.canton.crypto.store.memory.{ InMemoryCryptoPublicStore, } import com.digitalasset.canton.logging.{NamedLoggerFactory, TracedLogger} -import com.digitalasset.canton.tracing.TraceContext import com.google.protobuf.ByteString import com.typesafe.scalalogging.LazyLogging import org.bouncycastle.asn1.x509.AlgorithmIdentifier -import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContext} +import java.security.{PrivateKey => JPrivateKey, PublicKey => JPublicKey} +import scala.concurrent.ExecutionContext object SymbolicCrypto extends LazyLogging { @@ -78,31 +76,14 @@ object SymbolicCrypto extends LazyLogging { def create( timeouts: ProcessingTimeout, loggerFactory: NamedLoggerFactory, - hkdfOps: Option[HkdfOps] = None, - autoInitialize: Boolean = true, ): Crypto = { implicit val ec: ExecutionContext = DirectExecutionContext(TracedLogger(logger)) - import TraceContext.Implicits.Empty._ - val pureCrypto = new SymbolicPureCrypto(hkdfOps) + val pureCrypto = new SymbolicPureCrypto() val cryptoPublicStore = new InMemoryCryptoPublicStore val cryptoPrivateStore = new InMemoryCryptoPrivateStore(loggerFactory) val privateCrypto = new SymbolicPrivateCrypto(pureCrypto, cryptoPrivateStore) - if (autoInitialize) { - // Auto initialize the private store's HMAC - Await.result( - privateCrypto - .initializeHmacSecret() - .valueOr(err => - throw new RuntimeException( - s"Failed to initialize private crypto with HMAC secret: $err" - ) - ), - 10.seconds, - ) - } - // Conversion to java keys is not supported by symbolic crypto val javaKeyConverter = new JavaKeyConverter { override def toJava(privateKey: PrivateKey): Either[JavaKeyConversionError, JPrivateKey] = @@ -141,13 +122,12 @@ object SymbolicCrypto extends LazyLogging { def tryCreate( signingFingerprints: Seq[Fingerprint], fingerprintSuffixes: Seq[String], - hkdfOps: Option[HkdfOps], timeouts: ProcessingTimeout, loggerFactory: NamedLoggerFactory, ): Crypto = { import com.digitalasset.canton.tracing.TraceContext.Implicits.Empty._ - val crypto = SymbolicCrypto.create(timeouts, loggerFactory, hkdfOps) + val crypto = SymbolicCrypto.create(timeouts, loggerFactory) // Create a keypair for each signing fingerprint signingFingerprints.foreach { k => diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicCryptoTest.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicCryptoTest.scala index b67fd3f69..b7d3746b5 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicCryptoTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicCryptoTest.scala @@ -10,7 +10,7 @@ import com.digitalasset.canton.crypto.{ EncryptionKeyScheme, EncryptionTest, HashAlgorithm, - HmacPrivateTest, + RandomTest, SigningKeyScheme, SigningTest, SymmetricKeyScheme, @@ -38,7 +38,7 @@ class SymbolicCryptoTest extends AsyncWordSpec with SigningTest with EncryptionTest - with HmacPrivateTest { + with RandomTest { "SymbolicCrypto" can { @@ -51,7 +51,7 @@ class SymbolicCryptoTest SymbolicCryptoProvider.supportedSymmetricKeySchemes, symbolicCrypto(), ) - behave like hmacProvider(symbolicCrypto().map(_.privateCrypto)) + behave like randomnessProvider(symbolicCrypto().map(_.pureCrypto)) // Symbolic crypto does not support Java key conversion, thus not tested } diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicPureCrypto.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicPureCrypto.scala index 12a95905c..c8bf1b8f0 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicPureCrypto.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/symbolic/SymbolicPureCrypto.scala @@ -10,9 +10,10 @@ import com.digitalasset.canton.serialization.{DeserializationError, Deterministi import com.digitalasset.canton.version.{HasVersionedToByteString, ProtocolVersion} import com.google.protobuf.ByteString -class SymbolicPureCrypto(hkdfOps: Option[HkdfOps] = None) extends CryptoPureApi { +class SymbolicPureCrypto() extends CryptoPureApi { private val symmetricKeyCounter = new AtomicInteger + private val randomnessCounter = new AtomicInteger // NOTE: The scheme is not really used by Symbolic crypto override val defaultSymmetricKeyScheme: SymmetricKeyScheme = SymmetricKeyScheme.Aes128Gcm @@ -214,14 +215,49 @@ class SymbolicPureCrypto(hkdfOps: Option[HkdfOps] = None) extends CryptoPureApi )(deserialize) } + override protected def computeHkdfInternal( + keyMaterial: ByteString, + outputBytes: Int, + info: HkdfInfo, + salt: ByteString, + algorithm: HmacAlgorithm, + ): Either[HkdfError, SecureRandomness] = + Right(SecureRandomness(keyMaterial.concat(salt).concat(info.bytes))) + + override protected def hkdfExpandInternal( + keyMaterial: SecureRandomness, + outputBytes: Int, + info: HkdfInfo, + algorithm: HmacAlgorithm, + ): Either[HkdfError, SecureRandomness] = + Right(SecureRandomness(keyMaterial.unwrap.concat(info.bytes))) + override def hkdfExpand( keyMaterial: SecureRandomness, outputBytes: Int, info: HkdfInfo, algorithm: HmacAlgorithm, ): Either[HkdfError, SecureRandomness] = - hkdfOps.fold(super.hkdfExpand(keyMaterial, outputBytes, info, algorithm))( - _.hkdfExpand(keyMaterial, outputBytes, info, algorithm) - ) + hkdfExpandInternal(keyMaterial, outputBytes, info, algorithm) + override def computeHkdf( + keyMaterial: ByteString, + outputBytes: Int, + info: HkdfInfo, + salt: ByteString, + algorithm: HmacAlgorithm, + ): Either[HkdfError, SecureRandomness] = + computeHkdfInternal(keyMaterial, outputBytes, info, salt, algorithm) + + override protected def generateRandomBytes(length: Int): Array[Byte] = { + // Not really random + val random = + DeterministicEncoding.encodeInt(randomnessCounter.getAndIncrement()).toByteArray.take(length) + + // Pad the rest of the request bytes with 0 if necessary + if (random.length < length) + random.concat(new Array[Byte](length - random.length)) + else + random + } } diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/tink/TinkCryptoTest.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/tink/TinkCryptoTest.scala index c65106b42..86d5961c9 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/tink/TinkCryptoTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/provider/tink/TinkCryptoTest.scala @@ -15,8 +15,8 @@ class TinkCryptoTest extends AsyncWordSpec with SigningTest with EncryptionTest - with HmacPrivateTest with HkdfTest + with RandomTest with JavaKeyConverterTest { "TinkCrypto" can { @@ -32,8 +32,8 @@ class TinkCryptoTest Tink.symmetric.supported, tinkCrypto(), ) - behave like hmacProvider(tinkCrypto().map(_.privateCrypto)) behave like hkdfProvider(tinkCrypto().map(_.pureCrypto)) + behave like randomnessProvider(tinkCrypto().map(_.pureCrypto)) // Tink provider does not support Java conversion of Ed25519 or Hybrid encryption keys behave like javaKeyConverterProvider( diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/store/CryptoPrivateStoreTest.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/store/CryptoPrivateStoreTest.scala index 1bd2098b8..b9aa7205b 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/store/CryptoPrivateStoreTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/store/CryptoPrivateStoreTest.scala @@ -12,9 +12,6 @@ trait CryptoPrivateStoreTest extends BaseTest { this: AsyncWordSpec => def cryptoPrivateStore(newStore: => CryptoPrivateStore): Unit = { - val secret1: HmacSecret = TestHmacSecret.generate() - val secret2: HmacSecret = TestHmacSecret.generate() - val sigKey1: SigningPrivateKey = SymbolicCrypto.signingPrivateKey("sigKey1") val sigKey1WithName: SigningPrivateKeyWithName = SigningPrivateKeyWithName(sigKey1, Some(KeyName.tryCreate("sigKey1"))) @@ -27,18 +24,6 @@ trait CryptoPrivateStoreTest extends BaseTest { this: AsyncWordSpec => val encKey2: EncryptionPrivateKey = SymbolicCrypto.encryptionPrivateKey("encKey2") val encKey2WithName: EncryptionPrivateKeyWithName = EncryptionPrivateKeyWithName(encKey2, None) - "store and retrieve HMAC secrets" in { - val store = newStore - for { - _ <- store.storeHmacSecret(secret1).valueOrFail("store hmac secret") - secret <- store.loadHmacSecret().valueOrFail("load hmac secret") - result <- store.hmacSecret.valueOrFail("retrieve hmac secret") - } yield { - secret shouldBe Some(secret1) - result shouldBe Some(secret1) - } - } - "store encryption keys correctly when added incrementally" in { val store = newStore for { @@ -123,15 +108,6 @@ trait CryptoPrivateStoreTest extends BaseTest { this: AsyncWordSpec => res.valueOr(err => fail(err.toString)) } - "rotate the HMAC secret" in { - val store = newStore - for { - _ <- store.storeHmacSecret(secret1).valueOrFail("store first hmac secret") - _ <- store.storeHmacSecret(secret2).valueOrFail("store second hmac secret") - } yield { - succeed - } - } } } diff --git a/community/common/src/test/scala/com/digitalasset/canton/crypto/store/db/DbCryptoPrivateStoreTest.scala b/community/common/src/test/scala/com/digitalasset/canton/crypto/store/db/DbCryptoPrivateStoreTest.scala index e172c1cc0..82fcefc71 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/crypto/store/db/DbCryptoPrivateStoreTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/crypto/store/db/DbCryptoPrivateStoreTest.scala @@ -18,8 +18,7 @@ trait DbCryptoPrivateStoreTest extends AsyncWordSpec with CryptoPrivateStoreTest storage.update( DBIO.seq( - sqlu"truncate table crypto_hmac_secret", - sqlu"truncate table crypto_private_keys", + sqlu"truncate table crypto_private_keys" ), operationName = s"${this.getClass}: Truncate private crypto tables", ) diff --git a/community/common/src/test/scala/com/digitalasset/canton/data/GenTransactionTreeTest.scala b/community/common/src/test/scala/com/digitalasset/canton/data/GenTransactionTreeTest.scala index 35459021e..317639eb9 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/data/GenTransactionTreeTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/data/GenTransactionTreeTest.scala @@ -4,7 +4,6 @@ package com.digitalasset.canton.data import com.daml.nonempty.{NonEmpty, NonEmptyUtil} -import com.digitalasset.canton.crypto.SecureRandomness import com.digitalasset.canton.{BaseTest, HasExecutionContext, LfPartyId} import com.digitalasset.canton.data.MerkleTree.RevealIfNeedBe import com.digitalasset.canton.topology.{ParticipantId, TestingIdentityFactory} @@ -115,7 +114,7 @@ class GenTransactionTreeTest extends AnyWordSpec with BaseTest with HasExecution ) shouldEqual Right(informeeTree) forAll(example.transactionTree.allLightTransactionViewTrees) { lt => - LightTransactionViewTree.fromByteString(example.hashOps)( + LightTransactionViewTree.fromByteString(example.cryptoOps)( lt.toByteString(ProtocolVersion.latestForTest) ) shouldBe Right(lt) } @@ -141,12 +140,16 @@ class GenTransactionTreeTest extends AnyWordSpec with BaseTest with HasExecution } "correctly reconstruct the top-level transaction view trees from the lightweight ones for each informee" in { - val seedLength = example.hashOps.defaultHashAlgorithm.length - val seed = SecureRandomness.secureRandomness(seedLength.toInt) + val seedLength = example.cryptoOps.defaultHashAlgorithm.length + val seed = example.cryptoOps.generateSecureRandomness(seedLength.toInt) val hkdfOps = ExampleTransactionFactory.hkdfOps val allLightTrees = example.transactionTree - .allLightTransactionViewTreesWithWitnessesAndSeeds(seed, hkdfOps) + .allLightTransactionViewTreesWithWitnessesAndSeeds( + seed, + hkdfOps, + ProtocolVersion.latestForTest, + ) .valueOrFail("Cant get the light transaction trees") val allTrees = example.transactionTree.allTransactionViewTrees.toList val allInformees = allLightTrees.map(_._1.informees).fold(Set.empty)(_.union(_)) diff --git a/community/common/src/test/scala/com/digitalasset/canton/data/MerkleTreeTest.scala b/community/common/src/test/scala/com/digitalasset/canton/data/MerkleTreeTest.scala index 7fec2db4c..56fa7980d 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/data/MerkleTreeTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/data/MerkleTreeTest.scala @@ -29,7 +29,7 @@ class MerkleTreeTest extends AnyWordSpec with BaseTest { val singletonLeaf3: Leaf3 = Leaf3(3) def singletonLeafHash(index: Int): RootHash = RootHash { - val salt = TestSalt.generate(index) + val salt = TestSalt.generateSalt(index) val data = DeterministicEncoding.encodeInt(index) val hashBuilder = TestHash.build hashBuilder @@ -191,7 +191,7 @@ object MerkleTreeTest { with HasCryptographicEvidence { this: A => - override def salt: Salt = TestSalt.generate(index) + override def salt: Salt = TestSalt.generateSalt(index) override def getCryptographicEvidence: ByteString = DeterministicEncoding.encodeInt(index) diff --git a/community/common/src/test/scala/com/digitalasset/canton/data/TransactionViewTest.scala b/community/common/src/test/scala/com/digitalasset/canton/data/TransactionViewTest.scala index 1e76e3c1a..06996890a 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/data/TransactionViewTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/data/TransactionViewTest.scala @@ -4,11 +4,11 @@ package com.digitalasset.canton.data import com.daml.lf.value.Value -import com.digitalasset.canton.{BaseTest, HasExecutionContext} import com.digitalasset.canton.crypto.{HashOps, Salt} import com.digitalasset.canton.protocol._ import com.digitalasset.canton.util.LfTransactionBuilder import com.digitalasset.canton.util.ShowUtil._ +import com.digitalasset.canton.{BaseTest, HasExecutionContext} import org.scalatest.wordspec.AnyWordSpec class TransactionViewTest extends AnyWordSpec with BaseTest with HasExecutionContext { @@ -27,7 +27,7 @@ class TransactionViewTest extends AnyWordSpec with BaseTest with HasExecutionCon ) val absoluteId: LfContractId = ExampleTransactionFactory.suffixedId(0, 0) val otherAbsoluteId: LfContractId = ExampleTransactionFactory.suffixedId(1, 1) - val salt: Salt = factory.transactionSeed + val salt: Salt = factory.transactionSalt val nodeSeed = ExampleTransactionFactory.lfHash(1) val globalKey: LfGlobalKey = LfGlobalKey(LfTransactionBuilder.defaultTemplateId, Value.ValueInt64(100L)) diff --git a/community/common/src/test/scala/com/digitalasset/canton/protocol/ExampleTransaction.scala b/community/common/src/test/scala/com/digitalasset/canton/protocol/ExampleTransaction.scala index 6603f9b01..1028e261d 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/protocol/ExampleTransaction.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/protocol/ExampleTransaction.scala @@ -5,7 +5,7 @@ package com.digitalasset.canton.protocol import cats.syntax.functor._ import com.digitalasset.canton.LfPartyId -import com.digitalasset.canton.crypto.HashOps +import com.digitalasset.canton.crypto.{HashOps, RandomOps} import com.digitalasset.canton.data._ import com.digitalasset.canton.protocol.WellFormedTransaction.{WithSuffixes, WithoutSuffixes} @@ -13,7 +13,7 @@ import com.digitalasset.canton.protocol.WellFormedTransaction.{WithSuffixes, Wit */ trait ExampleTransaction { - def hashOps: HashOps + def cryptoOps: HashOps with RandomOps /** Set of parties who are informees of an action (root or not) in the transaction */ def allInformees: Set[LfPartyId] = fullInformeeTree.allInformees diff --git a/community/common/src/test/scala/com/digitalasset/canton/protocol/ExampleTransactionFactory.scala b/community/common/src/test/scala/com/digitalasset/canton/protocol/ExampleTransactionFactory.scala index 5899ce693..bbd5c1564 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/protocol/ExampleTransactionFactory.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/protocol/ExampleTransactionFactory.scala @@ -355,9 +355,10 @@ object ExampleTransactionFactory { * Also provides convenience methods for creating [[ExampleTransaction]]s and parts thereof. */ class ExampleTransactionFactory( - val cryptoOps: HashOps with HmacOps with HkdfOps = new SymbolicPureCrypto + val cryptoOps: HashOps with HmacOps with HkdfOps with RandomOps = new SymbolicPureCrypto )( - val transactionSeed: Salt = TestSalt.generate(0), + val transactionSalt: Salt = TestSalt.generateSalt(0), + val transactionSeed: SaltSeed = TestSalt.generateSeed(0), val transactionUuid: UUID = UUID.fromString("11111111-2222-3333-4444-555555555555"), val confirmationPolicy: ConfirmationPolicy = ConfirmationPolicy.Signatory, val domainId: DomainId = DomainId(UniqueIdentifier.tryFromProtoPrimitive("example::default")), @@ -627,7 +628,7 @@ class ExampleTransactionFactory( case object EmptyTransaction extends ExampleTransaction { - override def hashOps: HashOps = ExampleTransactionFactory.this.cryptoOps + override def cryptoOps: HashOps with RandomOps = ExampleTransactionFactory.this.cryptoOps override def toString: String = "empty transaction" @@ -663,7 +664,7 @@ class ExampleTransactionFactory( } abstract class SingleNode(val nodeSeed: Option[LfHash]) extends ExampleTransaction { - override def hashOps: HashOps = ExampleTransactionFactory.this.cryptoOps + override def cryptoOps: HashOps with RandomOps = ExampleTransactionFactory.this.cryptoOps def lfContractId: LfContractId @@ -926,7 +927,7 @@ class ExampleTransactionFactory( */ case object MultipleRoots extends ExampleTransaction { - override def hashOps: HashOps = ExampleTransactionFactory.this.cryptoOps + override def cryptoOps: HashOps with RandomOps = ExampleTransactionFactory.this.cryptoOps override def toString: String = "multiple roots" @@ -1036,7 +1037,7 @@ class ExampleTransactionFactory( */ case object MultipleRootsAndViewNestings extends ExampleTransaction { - override def hashOps: HashOps = ExampleTransactionFactory.this.cryptoOps + override def cryptoOps: HashOps with RandomOps = ExampleTransactionFactory.this.cryptoOps override def toString: String = "transaction with multiple roots and view nestings" @@ -1479,7 +1480,7 @@ class ExampleTransactionFactory( */ case object ViewInterleavings extends ExampleTransaction { - override def hashOps: HashOps = ExampleTransactionFactory.this.cryptoOps + override def cryptoOps: HashOps with RandomOps = ExampleTransactionFactory.this.cryptoOps override def toString: String = "transaction with subviews and core nodes interleaved" @@ -2019,7 +2020,7 @@ class ExampleTransactionFactory( */ case object TransientContracts extends ExampleTransaction { - override def hashOps: HashOps = ExampleTransactionFactory.this.cryptoOps + override def cryptoOps: HashOps with RandomOps = ExampleTransactionFactory.this.cryptoOps override def toString: String = "transaction with transient contracts" diff --git a/community/common/src/test/scala/com/digitalasset/canton/protocol/messages/AcsCommitmentTest.scala b/community/common/src/test/scala/com/digitalasset/canton/protocol/messages/AcsCommitmentTest.scala index c05483517..edf4eb5db 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/protocol/messages/AcsCommitmentTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/protocol/messages/AcsCommitmentTest.scala @@ -10,6 +10,7 @@ import org.scalatest.wordspec.AnyWordSpec import com.digitalasset.canton.crypto.LtHash16 import com.digitalasset.canton.crypto.provider.symbolic.SymbolicPureCrypto import com.digitalasset.canton.data.CantonTimestamp +import com.digitalasset.canton.protocol.TestDomainParameters import com.digitalasset.canton.serialization.HasCryptographicEvidenceTest import com.digitalasset.canton.time.PositiveSeconds @@ -41,6 +42,7 @@ class AcsCommitmentTest extends AnyWordSpec with BaseTest with HasCryptographicE counterParticipant, period1, cmt, + TestDomainParameters.defaultStatic.protocolVersion, ) val commitment2 = AcsCommitment @@ -50,6 +52,7 @@ class AcsCommitmentTest extends AnyWordSpec with BaseTest with HasCryptographicE counterParticipant, period2, cmt, + TestDomainParameters.defaultStatic.protocolVersion, ) def fromByteString(bytes: ByteString): AcsCommitment = { diff --git a/community/common/src/test/scala/com/digitalasset/canton/protocol/messages/MediatorResponseTest.scala b/community/common/src/test/scala/com/digitalasset/canton/protocol/messages/MediatorResponseTest.scala index e37bbd9c4..8133a1d85 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/protocol/messages/MediatorResponseTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/protocol/messages/MediatorResponseTest.scala @@ -10,6 +10,7 @@ import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.topology.{DomainId, UniqueIdentifier} import com.digitalasset.canton.protocol.{RequestId, RootHash, ViewHash} import com.digitalasset.canton.serialization.HasCryptographicEvidenceTest +import com.digitalasset.canton.version.ProtocolVersion import com.digitalasset.canton.{BaseTest, LfPartyId, topology} import com.google.protobuf.ByteString import org.scalatest.wordspec.AnyWordSpec @@ -24,6 +25,7 @@ class MediatorResponseTest extends AnyWordSpec with BaseTest with HasCryptograph Some(RootHash(TestHash.digest("txid1"))), Set(LfPartyId.assertFromString("p1"), LfPartyId.assertFromString("p2")), DomainId(UniqueIdentifier.tryFromProtoPrimitive("da::default")), + ProtocolVersion.latestForTest, ) val response2: MediatorResponse = MediatorResponse.tryCreate( RequestId(CantonTimestamp.now()), @@ -33,6 +35,7 @@ class MediatorResponseTest extends AnyWordSpec with BaseTest with HasCryptograph Some(RootHash(TestHash.digest("txid3"))), Set.empty, DomainId(UniqueIdentifier.tryFromProtoPrimitive("da::default")), + ProtocolVersion.latestForTest, ) def fromByteString(bytes: ByteString): MediatorResponse = { diff --git a/community/common/src/test/scala/com/digitalasset/canton/sequencing/authentication/grpc/AuthenticationTokenManagerTest.scala b/community/common/src/test/scala/com/digitalasset/canton/sequencing/authentication/grpc/AuthenticationTokenManagerTest.scala index 01b79e1d9..4eb645577 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/sequencing/authentication/grpc/AuthenticationTokenManagerTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/sequencing/authentication/grpc/AuthenticationTokenManagerTest.scala @@ -6,6 +6,7 @@ package com.digitalasset.canton.sequencing.authentication.grpc import cats.data.EitherT import cats.implicits._ import com.digitalasset.canton.BaseTest +import com.digitalasset.canton.crypto.provider.symbolic.SymbolicPureCrypto import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.lifecycle.FutureUnlessShutdown import com.digitalasset.canton.sequencing.authentication.{ @@ -28,8 +29,9 @@ object AuthenticationTokenManagerTest extends org.mockito.MockitoSugar with Argu class AuthenticationTokenManagerTest extends AsyncWordSpec with BaseTest { - val token1: AuthenticationToken = AuthenticationToken.generate() - val token2: AuthenticationToken = AuthenticationToken.generate() + val crypto = new SymbolicPureCrypto + val token1: AuthenticationToken = AuthenticationToken.generate(crypto) + val token2: AuthenticationToken = AuthenticationToken.generate(crypto) val now = CantonTimestamp.Epoch "first call to getToken will obtain it" in { diff --git a/community/common/src/test/scala/com/digitalasset/canton/sequencing/authentication/grpc/SequencerClientAuthenticationTest.scala b/community/common/src/test/scala/com/digitalasset/canton/sequencing/authentication/grpc/SequencerClientAuthenticationTest.scala index 6d0c38732..f834a847e 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/sequencing/authentication/grpc/SequencerClientAuthenticationTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/sequencing/authentication/grpc/SequencerClientAuthenticationTest.scala @@ -5,6 +5,8 @@ package com.digitalasset.canton.sequencing.authentication.grpc import cats.data.EitherT import cats.implicits._ +import com.digitalasset.canton.BaseTest +import com.digitalasset.canton.crypto.provider.symbolic.SymbolicPureCrypto import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.domain.api.v0 import com.digitalasset.canton.domain.api.v0.{Hello, HelloServiceGrpc} @@ -14,7 +16,6 @@ import com.digitalasset.canton.sequencing.authentication.{ AuthenticationTokenManagerConfig, } import com.digitalasset.canton.topology.{DefaultTestIdentities, DomainId, UniqueIdentifier} -import com.digitalasset.canton.BaseTest import io.grpc._ import io.grpc.inprocess.{InProcessChannelBuilder, InProcessServerBuilder} import io.grpc.stub.StreamObserver @@ -28,8 +29,9 @@ class SequencerClientAuthenticationTest extends FixtureAsyncWordSpec with BaseTe val domainId = DomainId(UniqueIdentifier.tryFromProtoPrimitive("test::domain")) val participantId = DefaultTestIdentities.participant1 - val token1 = AuthenticationToken.generate() - val token2 = AuthenticationToken.generate() + val crypto = new SymbolicPureCrypto + val token1 = AuthenticationToken.generate(crypto) + val token2 = AuthenticationToken.generate(crypto) require(token1 != token2, "The generated tokens must be different") diff --git a/community/common/src/test/scala/com/digitalasset/canton/sequencing/protocol/SequencedEventTest.scala b/community/common/src/test/scala/com/digitalasset/canton/sequencing/protocol/SequencedEventTest.scala index 5555d6e30..62cdfe24b 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/sequencing/protocol/SequencedEventTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/sequencing/protocol/SequencedEventTest.scala @@ -27,6 +27,7 @@ class SequencedEventTest extends BaseTestWordSpec { Set.empty, TransferInDomainId(domainId), Verdict.Timeout, + ProtocolVersion.latestForTest, ), SymbolicCrypto.emptySignature, ) diff --git a/community/common/src/test/scala/com/digitalasset/canton/store/SequencedEventStoreTest.scala b/community/common/src/test/scala/com/digitalasset/canton/store/SequencedEventStoreTest.scala index ab8c6b5a0..c600f6564 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/store/SequencedEventStoreTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/store/SequencedEventStoreTest.scala @@ -27,7 +27,7 @@ trait SequencedEventStoreTest extends PrunableByTimeTest { val sequencerKey: Fingerprint = Fingerprint.tryCreate("sequencer") val crypto: Crypto = - SymbolicCrypto.tryCreate(Seq(sequencerKey), Seq(), None, timeouts, loggerFactory) + SymbolicCrypto.tryCreate(Seq(sequencerKey), Seq(), timeouts, loggerFactory) def sign(str: String): Signature = DefaultProcessingTimeouts.default diff --git a/community/common/src/test/scala/com/digitalasset/canton/store/db/DatabaseDeadlockTest.scala b/community/common/src/test/scala/com/digitalasset/canton/store/db/DatabaseDeadlockTest.scala index 00b698367..f012d6867 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/store/db/DatabaseDeadlockTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/store/db/DatabaseDeadlockTest.scala @@ -26,7 +26,7 @@ trait DatabaseDeadlockTest import rawStorage.api._ val batchSize = 100 - val roundsNegative = 10 + val roundsNegative = 50 val roundsPositive = 1 val maxRetries = 3 diff --git a/community/common/src/test/scala/com/digitalasset/canton/topology/TestingIdentityFactory.scala b/community/common/src/test/scala/com/digitalasset/canton/topology/TestingIdentityFactory.scala index 3d3c7f26d..953b2d2af 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/topology/TestingIdentityFactory.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/topology/TestingIdentityFactory.scala @@ -84,7 +84,6 @@ case class TestingTopology( mediators: Set[MediatorId] = Set(DefaultTestIdentities.mediator), additionalParticipants: Set[ParticipantId] = Set.empty, keyPurposes: Set[KeyPurpose] = KeyPurpose.all, - hkdfOps: Option[HkdfOps] = None, domainParameters: List[DynamicDomainParameters.WithValidity] = List( DynamicDomainParameters.WithValidity( validFrom = CantonTimestamp.Epoch, @@ -155,10 +154,6 @@ case class TestingTopology( ): TestingTopology = this.copy(topology = parties.fmap(w => w.fmap(ParticipantAttributes(_, trustLevel)))) - def withHkdfOps(hkdfOps: Option[HkdfOps]): TestingTopology = { - this.copy(hkdfOps = hkdfOps) - } - def build( loggerFactory: NamedLoggerFactory = NamedLoggerFactory("test-area", "crypto") ): TestingIdentityFactory = @@ -177,7 +172,7 @@ class TestingIdentityFactory( new SyncCryptoApiProvider( owner, ips(), - newCrypto(owner, hkdfOps = topology.hkdfOps), + newCrypto(owner), CachingConfigs.testing, DefaultProcessingTimeouts.testing, loggerFactory, @@ -391,7 +386,6 @@ class TestingIdentityFactory( owner: KeyOwner, signingFingerprints: Seq[Fingerprint] = Seq(), fingerprintSuffixes: Seq[String] = Seq(), - hkdfOps: Option[HkdfOps] = None, ): Crypto = { val signingFingerprintsOrOwner = if (signingFingerprints.isEmpty) @@ -408,7 +402,6 @@ class TestingIdentityFactory( SymbolicCrypto.tryCreate( signingFingerprintsOrOwner, fingerprintSuffixesOrOwner, - hkdfOps, DefaultProcessingTimeouts.testing, loggerFactory, ) @@ -522,8 +515,9 @@ class TestingOwnerWithKeys( val dpc1Updated = mkDmGov( DomainParametersChange( DomainId(uid), - defaultDomainParameters.copy(participantResponseTimeout = - NonNegativeFiniteDuration.ofSeconds(2) + defaultDomainParameters.copy( + participantResponseTimeout = NonNegativeFiniteDuration.ofSeconds(2), + topologyChangeDelay = NonNegativeFiniteDuration.ofMillis(100), ), ), namespaceKey, diff --git a/community/common/src/test/scala/com/digitalasset/canton/topology/processing/TopologyTimestampPlusEpsilonTrackerTest.scala b/community/common/src/test/scala/com/digitalasset/canton/topology/processing/TopologyTimestampPlusEpsilonTrackerTest.scala index cc1a1999e..1b9860090 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/topology/processing/TopologyTimestampPlusEpsilonTrackerTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/topology/processing/TopologyTimestampPlusEpsilonTrackerTest.scala @@ -9,31 +9,42 @@ import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.lifecycle.FutureUnlessShutdown import com.digitalasset.canton.protocol.DynamicDomainParameters import com.digitalasset.canton.time.NonNegativeFiniteDuration +import com.digitalasset.canton.topology.store.ValidatedTopologyTransaction import com.digitalasset.canton.topology.store.memory.InMemoryTopologyStore -import com.digitalasset.canton.topology.transaction.DomainParametersChange +import com.digitalasset.canton.topology.transaction.{ + DomainParametersChange, + SignedTopologyTransaction, + TopologyChangeOp, +} import com.digitalasset.canton.topology.{DefaultTestIdentities, TestingOwnerWithKeys} -import com.digitalasset.canton.{BaseTestWordSpec, HasExecutionContext} +import com.digitalasset.canton.{BaseTest, HasExecutionContext} +import org.scalatest.Outcome +import org.scalatest.wordspec.FixtureAnyWordSpec import scala.concurrent.Future -class TopologyTimestampPlusEpsilonTrackerTest extends BaseTestWordSpec with HasExecutionContext { +class TopologyTimestampPlusEpsilonTrackerTest + extends FixtureAnyWordSpec + with BaseTest + with HasExecutionContext { import com.digitalasset.canton.topology.client.EffectiveTimeTestHelpers._ - private def setup( - prepareO: Option[(CantonTimestamp, NonNegativeFiniteDuration)] = Some( - (ts.immediatePredecessor, epsilonFD) - ) - ): (TopologyTimestampPlusEpsilonTracker, InMemoryTopologyStore) = { + protected class Fixture { + val crypto = new TestingOwnerWithKeys( + DefaultTestIdentities.domainManager, + loggerFactory, + parallelExecutionContext, + ) val store = new InMemoryTopologyStore(loggerFactory) val tracker = new TopologyTimestampPlusEpsilonTracker(DefaultProcessingTimeouts.testing, loggerFactory) - prepareO.foreach { case (ts, topologyChangeDelay) => - val crypto = new TestingOwnerWithKeys( - DefaultTestIdentities.domainManager, - loggerFactory, - parallelExecutionContext, - ) + + def appendEps( + sequenced: SequencedTime, + effective: EffectiveTime, + topologyChangeDelay: NonNegativeFiniteDuration, + ): Unit = { val tx = crypto.mkDmGov( DomainParametersChange( DefaultTestIdentities.domainId, @@ -41,19 +52,44 @@ class TopologyTimestampPlusEpsilonTrackerTest extends BaseTestWordSpec with HasE ), crypto.SigningKeys.key1, ) + append(sequenced, effective, tx) + } + + def append( + sequenced: SequencedTime, + effective: EffectiveTime, + txs: SignedTopologyTransaction[TopologyChangeOp]* + ): Unit = { + logger.debug(s"Storing $sequenced $effective $txs") store - .updateState( - SequencedTime(CantonTimestamp.MinValue), - EffectiveTime(CantonTimestamp.MinValue), - Seq(), - positive = Seq(tx), + .append( + sequenced, + effective, + txs.map(ValidatedTopologyTransaction(_, None)).toList, ) .futureValue - unwrap( - TopologyTimestampPlusEpsilonTracker.initializeFromStore(tracker, store, ts) - ).futureValue } - (tracker, store) + + def initTracker(ts: CantonTimestamp): Unit = { + unwrap(TopologyTimestampPlusEpsilonTracker.initialize(tracker, store, ts)).futureValue + } + + def init(): Unit = { + val myTs = ts.immediatePredecessor + val topologyChangeDelay = epsilonFD + appendEps( + SequencedTime(myTs), + EffectiveTime(myTs), + topologyChangeDelay, + ) + initTracker(myTs) + } + } + + type FixtureParam = Fixture + + override protected def withFixture(test: OneArgTest): Outcome = { + test(new Fixture) } private def unwrap[T](fut: FutureUnlessShutdown[T]): Future[T] = @@ -80,6 +116,7 @@ class TopologyTimestampPlusEpsilonTrackerTest extends BaseTestWordSpec with HasE val adjustedTs = unwrap(tracker.adjustTimestampForUpdate(ts)).futureValue tracker.adjustEpsilon(adjustedTs, ts, newEpsilon) + tracker.effectiveTimeProcessed(adjustedTs) // until adjustedTs, we should still get the old epsilon forAll(Seq(1L, 100L, 250L)) { delta => @@ -91,9 +128,9 @@ class TopologyTimestampPlusEpsilonTrackerTest extends BaseTestWordSpec with HasE "timestamp plus epsilon" should { - "epsilon is constant properly project the timestamp" in { - val (tracker, _) = setup() - + "epsilon is constant properly project the timestamp" in { f => + import f._ + init() assertEffectiveTimeForUpdate(tracker, ts, ts.plus(epsilon)) assertEffectiveTimeForUpdate(tracker, ts.plusSeconds(5), ts.plusSeconds(5).plus(epsilon)) unwrap( @@ -103,8 +140,9 @@ class TopologyTimestampPlusEpsilonTrackerTest extends BaseTestWordSpec with HasE } - "properly project during epsilon increase" in { - val (tracker, _) = setup() + "properly project during epsilon increase" in { f => + import f._ + init() val newEpsilon = NonNegativeFiniteDuration.ofSeconds(1) adjustEpsilon(tracker, newEpsilon) // after adjusted ts, we should get the new epsilon @@ -114,8 +152,9 @@ class TopologyTimestampPlusEpsilonTrackerTest extends BaseTestWordSpec with HasE } } - "properly deal with epsilon decrease" in { - val (tracker, _) = setup() + "properly deal with epsilon decrease" in { f => + import f._ + init() val newEpsilon = NonNegativeFiniteDuration.ofMillis(100) adjustEpsilon(tracker, newEpsilon) @@ -126,9 +165,39 @@ class TopologyTimestampPlusEpsilonTrackerTest extends BaseTestWordSpec with HasE } } - "gracefully deal with epsilon decrease on a buggy domain" in { + "correctly initialise with pending changes" in { f => + import f._ + + val eS1 = NonNegativeFiniteDuration.ofMillis(100) + val tsS1 = ts.minusSeconds(1) + appendEps(SequencedTime(tsS1), EffectiveTime(tsS1), eS1) + + val eM100 = NonNegativeFiniteDuration.ofMillis(100) + val tsM100 = ts.minusMillis(100) + + appendEps(SequencedTime(tsM100), EffectiveTime(ts), eM100) + + val tsM75 = ts.minusMillis(75) + append( + SequencedTime(ts), + EffectiveTime(tsM75.plusMillis(100)), + f.crypto.TestingTransactions.ns1k2, + f.crypto.TestingTransactions.p2p1, + ) + + val eM50 = NonNegativeFiniteDuration.ofMillis(200) + val tsM50 = ts.minusMillis(50) + appendEps(SequencedTime(tsM50), EffectiveTime(tsM50.plusMillis(100)), eM50) + + f.initTracker(ts) + + assertEffectiveTimeForUpdate(tracker, ts.plusMillis(10), ts.plusMillis(110)) + + } - val (tracker, _) = setup() + "gracefully deal with epsilon decrease on a buggy domain" in { f => + import f._ + init() val newEpsilon = NonNegativeFiniteDuration.ofMillis(100) adjustEpsilon(tracker, newEpsilon) @@ -155,13 +224,17 @@ class TopologyTimestampPlusEpsilonTrackerTest extends BaseTestWordSpec with HasE } } - "block until we are in sync again" in { - val (tracker, _) = setup() + "block until we are in sync again" in { f => + import f._ + init() val eps3 = epsilon.dividedBy(3) val eps2 = epsilon.dividedBy(2) epsilon.toMillis shouldBe 250 // this test assumes this - // first, we'll kick off the computation + // first, we'll kick off the computation at ts (note epsilon is active for ts on) val fut1 = unwrap(tracker.adjustTimestampForUpdate(ts)).futureValue + // now, we need to confirm this effective time, which will "unlock" the any sequenced event update + // within ts + eps + tracker.effectiveTimeProcessed(fut1) // then, we tick another update with a third and with half epsilon val fut2 = unwrap(tracker.adjustTimestampForUpdate(ts.plus(eps3))).futureValue val fut3 = unwrap(tracker.adjustTimestampForUpdate(ts.plus(eps2))).futureValue diff --git a/community/common/src/test/scala/com/digitalasset/canton/topology/store/TopologyStoreTest.scala b/community/common/src/test/scala/com/digitalasset/canton/topology/store/TopologyStoreTest.scala index 31891812b..d5dadae84 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/topology/store/TopologyStoreTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/topology/store/TopologyStoreTest.scala @@ -17,8 +17,10 @@ import org.scalatest.{Assertion, BeforeAndAfterAll} import scala.annotation.nowarn import scala.concurrent.Future import cats.syntax.contravariantSemigroupal._ +import com.digitalasset.canton.time.NonNegativeFiniteDuration import com.digitalasset.canton.topology.processing.{EffectiveTime, SequencedTime} import com.digitalasset.canton.topology.transaction.TopologyChangeOp.{Add, Replace} +import com.digitalasset.canton.util.MonadUtil trait TopologyStoreTest extends AsyncWordSpec @@ -311,7 +313,7 @@ trait TopologyStoreTest ) ) currentTs <- store.timestamp() - } yield currentTs shouldBe Some(ts.plusSeconds(5)) + } yield currentTs.map(_._2.value) shouldBe Some(ts.plusSeconds(5)) } "successfully append new items" in { @@ -461,10 +463,10 @@ trait TopologyStoreTest "", false, ) - _ <- store.findActiveTransactionsForMapping(okm1.transaction.element.mapping) + _ <- store.findPositiveTransactionsForMapping(okm1.transaction.element.mapping) _ <- store.findRemovalTransactionForMappings(Set(okm1.transaction.element)) _ <- store.findDispatchingTransactionsAfter(ts1) - _ <- store.findEffectiveTimestampsSince(ts1) + _ <- store.findUpcomingEffectiveChanges(ts1) _ <- store.findTsOfParticipantStateChangesBefore(ts1, participant1, 100) _ <- store.findTransactionsInRange(ts, ts1) } yield { @@ -544,12 +546,12 @@ trait TopologyStoreTest snapshot1 <- snapshot(ts.plusMillis(5)) snapshot1b <- snapshot(ts1.immediatePredecessor) snapshot2 <- snapshot(ts1.plusMillis(5)) - empty2 <- store.findActiveTransactionsForMapping(okm1.transaction.element.mapping) - activeOkm2 <- store.findActiveTransactionsForMapping(okm2.transaction.element.mapping) + empty2 <- store.findPositiveTransactionsForMapping(okm1.transaction.element.mapping) + activeOkm2 <- store.findPositiveTransactionsForMapping(okm2.transaction.element.mapping) foundRokm1 <- store.findRemovalTransactionForMappings(Set(okm1.transaction.element)) _ <- testQueriesUsedForDomainTopologyDispatching() } yield { - maxTimestamp should contain(ts1) + maxTimestamp.map(_._2.value) should contain(ts1) all.result should have length ((first.length + snd.length - 1).toLong) headState.result.map(_.transaction) shouldBe Seq(ns1k1, ns1k2, id1k1, ps1, dpc1, okm2) empty1 shouldBe (Nil, Nil) @@ -562,6 +564,67 @@ trait TopologyStoreTest } } + "test bootstrapping queries" in { + val store = mk() + + // we have the following changes + val changes = List( + (0, 0) -> Seq(ns1k1), + (100, 100) -> Seq(ns1k2, dpc1), // epsilon 250ms + (110, 260) -> Seq(id1k1), + (120, 270) -> Seq(dpc1Updated), + (150, 300) -> Seq(okm1), + ) + + implicit class AddMs(ts: CantonTimestamp) { + def +(ms: Int): CantonTimestamp = ts.plusMillis(ms.toLong) + } + + val storeF = MonadUtil.sequentialTraverse_(changes) { + case ((sequenced, effective), items) => + store.append( + SequencedTime(ts + sequenced), + EffectiveTime(ts + effective), + items.map(toValidated), + ) + } + + def change(sequenced: Int, effective: Int, dpc: Option[Int]): TopologyStore.Change = { + dpc.fold( + TopologyStore.Change + .Other( + SequencedTime(ts + sequenced), + EffectiveTime(ts + effective), + ): TopologyStore.Change + ) { epsilon => + TopologyStore.Change.TopologyDelay( + SequencedTime(ts + sequenced), + EffectiveTime(ts + effective), + NonNegativeFiniteDuration.ofMillis(epsilon.toLong), + ) + } + } + + for { + _ <- storeF + empty1 <- store.findUpcomingEffectiveChanges(ts + 301) + last1 <- store.findUpcomingEffectiveChanges(ts + 271) + pendingDpc <- store.findUpcomingEffectiveChanges(ts + 270) + firstDpc <- store.findUpcomingEffectiveChanges(ts + 101) + timestamps <- store.timestamp(useStateStore = false) + } yield { + val change150 = change(150, 300, None) + val change120 = change(120, 270, Some(100)) + val change110 = change(110, 260, None) + empty1 shouldBe empty + last1 shouldBe Seq(change150) + pendingDpc shouldBe Seq(change120, change150) + firstDpc shouldBe Seq(change110, change120, change150) + timestamps should contain((SequencedTime(ts + 150), EffectiveTime(ts + 300))) + } + + } + "namespace filter behaves correctly" in { val store = mk() import factory.SigningKeys._ diff --git a/community/common/src/test/scala/com/digitalasset/canton/util/retry/PolicyTest.scala b/community/common/src/test/scala/com/digitalasset/canton/util/retry/PolicyTest.scala index 7f7f8a4de..48f738cc0 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/util/retry/PolicyTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/util/retry/PolicyTest.scala @@ -668,7 +668,9 @@ class PolicyTest extends AsyncFunSpec with BaseTest with HasExecutorService { val retried = new AtomicInteger() def run(): Future[Int] = Future.successful { - retried.incrementAndGet() + val num = retried.incrementAndGet() + logger.debug(s"Increment retried is ${num}, closeable is ${closeable.isClosing}") + num } val retryF = @@ -677,15 +679,15 @@ class PolicyTest extends AsyncFunSpec with BaseTest with HasExecutorService { logger.debug(s"Stopped retry after $count") }(executorService) + logger.debug("Wrapping") // Wrap the retry in a performUnlessClosing to trigger possible deadlocks. val retryUnlessClosingF = closeable.performUnlessClosingF("test-retry")(retryF)(executorService, traceContext) Threading.sleep(10) - closeable.close() - inside(Await.result(retryUnlessClosingF.unwrap, 50.millis)) { + inside(Await.result(retryUnlessClosingF.unwrap, 100.millis)) { case UnlessShutdown.Outcome(_) => succeed case UnlessShutdown.AbortedDueToShutdown => fail("Unexpected shutdown.") } diff --git a/community/common/src/test/scala/com/digitalasset/canton/version/HasProtocolVersionedWrapperTest.scala b/community/common/src/test/scala/com/digitalasset/canton/version/HasProtocolVersionedWrapperTest.scala index 507691320..28c84841d 100644 --- a/community/common/src/test/scala/com/digitalasset/canton/version/HasProtocolVersionedWrapperTest.scala +++ b/community/common/src/test/scala/com/digitalasset/canton/version/HasProtocolVersionedWrapperTest.scala @@ -72,7 +72,7 @@ object HasProtocolVersionedWrapperTest { )( val deserializedFrom: Option[ByteString] = None ) extends HasProtocolVersionedWrapper[VersionedMessage[Message]] { - override def toProtoVersioned: VersionedMessage[Message] = Message.toProtoVersionedV2(this) + override def toProtoVersioned: VersionedMessage[Message] = Message.toProtoVersioned(this) override def getCryptographicEvidence: ByteString = super[HasProtocolVersionedWrapper].toByteString diff --git a/community/demo/src/main/daml/ai-analysis/daml.yaml b/community/demo/src/main/daml/ai-analysis/daml.yaml index e92ad748d..54e4ec491 100644 --- a/community/demo/src/main/daml/ai-analysis/daml.yaml +++ b/community/demo/src/main/daml/ai-analysis/daml.yaml @@ -1,4 +1,4 @@ -sdk-version: 2.2.0-snapshot.20220504.9851.0.4c8e027d +sdk-version: 2.3.0-snapshot.20220511.9887.0.5a509164 name: ai-analysis exposed-modules: - AIAnalysis diff --git a/community/demo/src/main/daml/bank/daml.yaml b/community/demo/src/main/daml/bank/daml.yaml index 94187726a..16f436cdd 100644 --- a/community/demo/src/main/daml/bank/daml.yaml +++ b/community/demo/src/main/daml/bank/daml.yaml @@ -1,4 +1,4 @@ -sdk-version: 2.2.0-snapshot.20220504.9851.0.4c8e027d +sdk-version: 2.3.0-snapshot.20220511.9887.0.5a509164 name: bank exposed-modules: - Bank diff --git a/community/demo/src/main/daml/doctor/daml.yaml b/community/demo/src/main/daml/doctor/daml.yaml index 89fba612d..8388e4ebb 100644 --- a/community/demo/src/main/daml/doctor/daml.yaml +++ b/community/demo/src/main/daml/doctor/daml.yaml @@ -1,4 +1,4 @@ -sdk-version: 2.2.0-snapshot.20220504.9851.0.4c8e027d +sdk-version: 2.3.0-snapshot.20220511.9887.0.5a509164 name: doctor exposed-modules: - Doctor diff --git a/community/demo/src/main/daml/health-insurance/daml.yaml b/community/demo/src/main/daml/health-insurance/daml.yaml index 49e4b62b0..2708482bd 100644 --- a/community/demo/src/main/daml/health-insurance/daml.yaml +++ b/community/demo/src/main/daml/health-insurance/daml.yaml @@ -1,4 +1,4 @@ -sdk-version: 2.2.0-snapshot.20220504.9851.0.4c8e027d +sdk-version: 2.3.0-snapshot.20220511.9887.0.5a509164 name: health-insurance exposed-modules: - HealthInsurance diff --git a/community/demo/src/main/daml/medical-records/daml.yaml b/community/demo/src/main/daml/medical-records/daml.yaml index f6b025592..9b9eb8408 100644 --- a/community/demo/src/main/daml/medical-records/daml.yaml +++ b/community/demo/src/main/daml/medical-records/daml.yaml @@ -1,4 +1,4 @@ -sdk-version: 2.2.0-snapshot.20220504.9851.0.4c8e027d +sdk-version: 2.3.0-snapshot.20220511.9887.0.5a509164 name: medical-records exposed-modules: - MedicalRecord diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/DomainNodeBootstrap.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/DomainNodeBootstrap.scala index b0da57774..e45018330 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/DomainNodeBootstrap.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/DomainNodeBootstrap.scala @@ -26,7 +26,11 @@ import com.digitalasset.canton.domain.admin.{grpc => admingrpc} import com.digitalasset.canton.domain.config._ import com.digitalasset.canton.domain.governance.ParticipantAuditor import com.digitalasset.canton.domain.initialization._ -import com.digitalasset.canton.domain.mediator.{DomainNodeMediatorFactory, MediatorRuntime} +import com.digitalasset.canton.domain.mediator.{ + CommunityMediatorRuntimeFactory, + MediatorRuntime, + MediatorRuntimeFactory, +} import com.digitalasset.canton.domain.metrics.DomainMetrics import com.digitalasset.canton.domain.sequencing.admin._ import com.digitalasset.canton.domain.sequencing.admin.client.SequencerAdminClient @@ -89,7 +93,7 @@ class DomainNodeBootstrap( legalIdentityHook: X509Certificate => EitherT[Future, String, Unit], addMemberHook: DomainTopologyManager.AddMemberHook, sequencerRuntimeFactory: SequencerRuntimeFactory, - mediatorFactory: DomainNodeMediatorFactory, + mediatorFactory: MediatorRuntimeFactory, storageFactory: StorageFactory, futureSupervisor: FutureSupervisor, )(implicit @@ -117,12 +121,16 @@ class DomainNodeBootstrap( legalIdentityHook, DynamicDomainParameters.initialValues(clock), ) - (nodeId, topologyManager, ns) = initialized + (nodeId, topologyManager, namespaceKey) = initialized domainId = DomainId(nodeId.identity) - _ <- initializeMediator(domainId, ns, topologyManager) + _ <- initializeMediator(domainId, namespaceKey, topologyManager) _ <- initializeSequencerServices(config.domainParameters.protocolVersion.unwrap) - _ <- initializeSequencer(domainId, topologyManager) - _ <- startIfDomainManagerReadyOrDefer(topologyManager, nodeId) + _ <- initializeSequencer(domainId, topologyManager, namespaceKey) + // finally, store the node id (which means we have completed initialisation) + // as all methods above are idempotent, if we die during initialisation, we should come back here + // and resume until we've stored the node id + _ <- storeId(nodeId) + _ <- startDomain(topologyManager) } yield () } @@ -138,20 +146,13 @@ class DomainNodeBootstrap( private def initializeMediator( domainId: DomainId, - namespaceKey: PublicKey, + namespaceKey: SigningPublicKey, topologyManager: DomainTopologyManager, ): EitherT[Future, String, Unit] = { // In a domain without a dedicated DomainTopologyManager, the mediator always gets the same ID as the domain. val mediatorId = MediatorId(domainId) for { - mediatorKey <- mediatorFactory.fetchInitialMediatorKey( - name, - domainId, - mediatorId, - namespaceKey, - topologyManager, - crypto, - ) + mediatorKey <- getOrCreateSigningKey(s"$name-mediator-signing") _ <- authorizeStateUpdate( topologyManager, namespaceKey, @@ -168,6 +169,7 @@ class DomainNodeBootstrap( private def initializeSequencer( domainId: DomainId, topologyManager: DomainTopologyManager, + namespaceKey: SigningPublicKey, ): EitherT[Future, String, PublicKey] = { def createAdminConnection(): EitherT[Future, String, SequencerAdminClient] = for { @@ -203,7 +205,7 @@ class DomainNodeBootstrap( parameters.sequencerClient, parameters.devVersionSupport, adminClient, - topologyManager, + authorizeStateUpdate(topologyManager, namespaceKey, _), crypto.cryptoPublicStore, request, parameters.processingTimeouts, @@ -260,7 +262,7 @@ class DomainNodeBootstrap( override protected def initialize(id: NodeId): EitherT[Future, String, Unit] = { val topologyManager = initializeIdentityManagerAndServices(id) - startIfDomainManagerReadyOrDefer(topologyManager, id) + startIfDomainManagerReadyOrDefer(topologyManager) } /** The Domain cannot be started until the domain manager has keys for all domain entities available. These keys @@ -276,8 +278,7 @@ class DomainNodeBootstrap( * I don't believe we currently have a means of doing this. */ private def startIfDomainManagerReadyOrDefer( - manager: DomainTopologyManager, - nodeId: NodeId, + manager: DomainTopologyManager ): EitherT[Future, String, Unit] = { def deferStart: EitherT[Future, String, Unit] = { val attemptedStart = new AtomicBoolean(false) @@ -300,7 +301,7 @@ class DomainNodeBootstrap( // we're now the top level error handler of starting a domain so log appropriately val domainStarted = initTimeout.await("Domain startup awaiting domain ready to handle requests")( - startDomain(manager, nodeId).value + startDomain(manager).value ) domainStarted match { case Left(error) => @@ -320,15 +321,12 @@ class DomainNodeBootstrap( // if the domain is starting up after previously running its identity will have been stored and will be immediately available alreadyInitialized <- EitherT.right[String](manager.isInitialized) // if not, then create an observer of topology transactions that will check each time whether full identity has been generated - _ <- if (alreadyInitialized) startDomain(manager, nodeId) else deferStart + _ <- if (alreadyInitialized) startDomain(manager) else deferStart } yield () } /** Attempt to create the domain and only return and call setInstance once it is ready to handle requests */ - private def startDomain( - manager: DomainTopologyManager, - nodeId: NodeId, - ): EitherT[Future, String, Unit] = + private def startDomain(manager: DomainTopologyManager): EitherT[Future, String, Unit] = startInstanceUnlessClosing { // store with all topology transactions which were timestamped and distributed via sequencer val domainId = DomainId(manager.id) @@ -463,7 +461,6 @@ class DomainNodeBootstrap( topologyManagementArtefacts <- TopologyManagementInitialization( config, domainId, - nodeId, storage, clock, crypto, @@ -484,8 +481,8 @@ class DomainNodeBootstrap( mediatorRuntime <- EmbeddedMediatorInitialization( domainId, - nodeId, parameters, + staticDomainParameters.protocolVersion, clock, crypto, topologyStoreFactory.forId(DomainStore(domainId, discriminator = "M")), @@ -572,7 +569,7 @@ object DomainNodeBootstrap { DomainTopologyManager.legalIdentityHookNoOp, DomainTopologyManager.addMemberNoOp, new SequencerRuntimeFactory.Community(config.sequencer), - DomainNodeMediatorFactory.Embedded, + CommunityMediatorRuntimeFactory, new CommunityStorageFactory(config.storage), futureSupervisor, ) diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/DomainTopologyManagerIdentityInitialization.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/DomainTopologyManagerIdentityInitialization.scala index 023fe19e8..3fbd385ed 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/DomainTopologyManagerIdentityInitialization.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/DomainTopologyManagerIdentityInitialization.scala @@ -6,7 +6,7 @@ package com.digitalasset.canton.domain.initialization import cats.data.EitherT import cats.syntax.either._ import com.digitalasset.canton.config.RequireTypes.InstanceName -import com.digitalasset.canton.crypto.{KeyName, PublicKey, X509Certificate} +import com.digitalasset.canton.crypto.{SigningPublicKey, X509Certificate} import com.digitalasset.canton.domain.config.DomainNodeParameters import com.digitalasset.canton.domain.topology.DomainTopologyManager import com.digitalasset.canton.environment.CantonNodeBootstrapBase @@ -16,7 +16,7 @@ import com.digitalasset.canton.topology.transaction.{ NamespaceDelegation, OwnerToKeyMapping, } -import com.digitalasset.canton.topology.{DomainId, _} +import com.digitalasset.canton.topology._ import com.digitalasset.canton.tracing.TraceContext import scala.concurrent.Future @@ -30,45 +30,52 @@ trait DomainTopologyManagerIdentityInitialization { initialDynamicDomainParameters: DynamicDomainParameters, )(implicit traceContext: TraceContext - ): EitherT[Future, String, (NodeId, DomainTopologyManager, PublicKey)] = + ): EitherT[Future, String, (NodeId, DomainTopologyManager, SigningPublicKey)] = // initialize domain with local keys for { id <- Identifier.create(name.unwrap).toEitherT[Future] - // TODO(i5202) if we crash here, we will like not be able to recover. - nsName <- KeyName.create(s"$name-namespace").toEitherT[Future] - ns <- crypto - .generateSigningKey(name = Some(nsName)) - .leftMap(err => s"Failed to generate namespace key: $err") - idmName <- KeyName.create(s"$name-signing").toEitherT[Future] - idmKey <- crypto - .generateSigningKey(name = Some(idmName)) - .leftMap(err => s"Failed to generate identity key: $err") - uid = UniqueIdentifier(id, Namespace(ns.fingerprint)) + // first, we create the namespace key for this node + namespaceKey <- getOrCreateSigningKey(s"$name-namespace") + // then, we create the topology manager signing key + topologyManagerSigningKey <- getOrCreateSigningKey(s"$name-topology-manager-signing") + // using the namespace key and the identifier, we create the uid of this node + uid = UniqueIdentifier(id, Namespace(namespaceKey.fingerprint)) nodeId = NodeId(uid) - _ <- storeId(nodeId) - - idMgr = initializeIdentityManagerAndServices(nodeId) - + // now, we kick off the topology manager services so we can start building topology transactions + topologyManager = initializeIdentityManagerAndServices(nodeId) + // first, we issue the root namespace delegation for our namespace _ <- authorizeStateUpdate( - idMgr, - ns, - NamespaceDelegation(Namespace(ns.fingerprint), ns, isRootDelegation = true), + topologyManager, + namespaceKey, + NamespaceDelegation( + Namespace(namespaceKey.fingerprint), + namespaceKey, + isRootDelegation = true, + ), ) - + // then, we initialise the domain parameters _ <- authorizeDomainGovernance( - idMgr, - ns, + topologyManager, + namespaceKey, DomainParametersChange(DomainId(uid), initialDynamicDomainParameters), ) + // now, we assign the topology manager key with the domain topology manager domainTopologyManagerId = DomainTopologyManagerId(uid) - _ <- authorizeStateUpdate(idMgr, ns, OwnerToKeyMapping(domainTopologyManagerId, idmKey)) + _ <- authorizeStateUpdate( + topologyManager, + namespaceKey, + OwnerToKeyMapping(domainTopologyManagerId, topologyManagerSigningKey), + ) // Setup the legal identity of the domain nodes - domainCert <- new LegalIdentityInit(certificateGenerator, crypto) - .generateCertificate(uid, Seq(MediatorId(uid), domainTopologyManagerId)) + domainCert <- (new LegalIdentityInit(certificateGenerator, crypto)).getOrGenerateCertificate( + uid, + Seq(MediatorId(uid), domainTopologyManagerId), + ) _ <- legalIdentityHook(domainCert) - } yield (nodeId, idMgr, ns) + + } yield (nodeId, topologyManager, namespaceKey) protected def initializeIdentityManagerAndServices(nodeId: NodeId): DomainTopologyManager diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/EmbeddedMediatorInitialization.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/EmbeddedMediatorInitialization.scala index 66f13e54e..81f313ddd 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/EmbeddedMediatorInitialization.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/EmbeddedMediatorInitialization.scala @@ -8,7 +8,7 @@ import cats.data.EitherT import com.digitalasset.canton.concurrent.FutureSupervisor import com.digitalasset.canton.crypto.{Crypto, DomainSyncCryptoClient} import com.digitalasset.canton.domain.config.DomainNodeParameters -import com.digitalasset.canton.domain.mediator.{DomainNodeMediatorFactory, MediatorRuntime} +import com.digitalasset.canton.domain.mediator.{MediatorRuntime, MediatorRuntimeFactory} import com.digitalasset.canton.domain.metrics.DomainMetrics import com.digitalasset.canton.logging.NamedLoggerFactory import com.digitalasset.canton.resource.Storage @@ -24,8 +24,9 @@ import com.digitalasset.canton.time.{Clock, DomainTimeTrackerConfig} import com.digitalasset.canton.topology.client.DomainTopologyClientWithInit import com.digitalasset.canton.topology.processing.TopologyTransactionProcessor import com.digitalasset.canton.topology.store.TopologyStore -import com.digitalasset.canton.topology.{DomainId, MediatorId, NodeId} +import com.digitalasset.canton.topology.{DomainId, MediatorId} import com.digitalasset.canton.tracing.TraceContext +import com.digitalasset.canton.version.ProtocolVersion import io.opentelemetry.api.trace.Tracer import scala.concurrent.{ExecutionContextExecutorService, Future} @@ -34,8 +35,8 @@ object EmbeddedMediatorInitialization { def apply( id: DomainId, - nodeId: NodeId, cantonParameterConfig: DomainNodeParameters, + protocolVersion: ProtocolVersion, clock: Clock, crypto: Crypto, mediatorTopologyStore: TopologyStore, @@ -43,7 +44,7 @@ object EmbeddedMediatorInitialization { storage: Storage, sequencerClientFactoryFactory: DomainTopologyClientWithInit => SequencerClientFactory, metrics: DomainMetrics, - domainNodeMediatorFactory: DomainNodeMediatorFactory, + mediatorFactory: MediatorRuntimeFactory, indexedStringStore: IndexedStringStore, futureSupervisor: FutureSupervisor, loggerFactory: NamedLoggerFactory, @@ -54,7 +55,6 @@ object EmbeddedMediatorInitialization { actorSystem: ActorSystem, ): EitherT[Future, String, MediatorRuntime] = { - val factory = domainNodeMediatorFactory.mediatorRuntimeFactory val timeouts = cantonParameterConfig.processingTimeouts val mediatorId = MediatorId(id) // The embedded mediator always has the same ID as the domain val sendTrackerStore = SendTrackerStore(storage) @@ -106,7 +106,7 @@ object EmbeddedMediatorInitialization { sequencedEventStore, sendTrackerStore, ) - mediatorRuntime <- factory + mediatorRuntime <- mediatorFactory .create( mediatorId, id, @@ -119,6 +119,7 @@ object EmbeddedMediatorInitialization { topologyProcessor, timeTrackerConfig, cantonParameterConfig, + protocolVersion, clock, metrics.mediator, futureSupervisor, diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/TopologyManagementInitialization.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/TopologyManagementInitialization.scala index 1ae815423..842afc816 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/TopologyManagementInitialization.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/initialization/TopologyManagementInitialization.scala @@ -100,7 +100,6 @@ object TopologyManagementInitialization { def apply( config: DomainBaseConfig, id: DomainId, - nodeId: NodeId, storage: Storage, clock: Clock, crypto: Crypto, diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/ConfirmationResponseProcessor.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/ConfirmationResponseProcessor.scala index 5acb9274b..bfad3090c 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/ConfirmationResponseProcessor.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/ConfirmationResponseProcessor.scala @@ -38,6 +38,7 @@ import com.digitalasset.canton.time.DomainTimeTracker import com.digitalasset.canton.tracing.{Spanning, TraceContext, Traced} import com.digitalasset.canton.util.ShowUtil._ import com.digitalasset.canton.util.{EitherTUtil, MonadUtil} +import com.digitalasset.canton.version.ProtocolVersion import com.google.common.annotations.VisibleForTesting import io.opentelemetry.api.trace.Tracer @@ -55,6 +56,7 @@ class ConfirmationResponseProcessor( timeTracker: DomainTimeTracker, val mediatorState: MediatorState, alarmer: AlarmStreamer, + protocolVersion: ProtocolVersion, protected val loggerFactory: NamedLoggerFactory, )(implicit ec: ExecutionContext, tracer: Tracer) extends NamedLogging @@ -316,7 +318,13 @@ class ConfirmationResponseProcessor( envs <- recipientsByViewType.toSeq .traverse { case (viewType, recipients) => val rejection = - MalformedMediatorRequestResult(requestId, domain, viewType, rejectionReason) + MalformedMediatorRequestResult( + requestId, + domain, + viewType, + rejectionReason, + protocolVersion, + ) SignedProtocolMessage .tryCreate(rejection, snapshot, crypto.pureCrypto) .map { signedRejection => @@ -512,14 +520,17 @@ class ConfirmationResponseProcessor( .Reject(show"$informeesNoParticipant") else verdict } - envelopes <- informeesMap - .map { case (participantId, informees) => - val result = request.createMediatorResult(requestId, verdictWithInformeeCheck, informees) + envelopes <- informeesMap.toList + .traverse { case (participantId, informees) => + val result = request.createMediatorResult( + requestId, + verdictWithInformeeCheck, + informees, + protocolVersion, + ) SignedProtocolMessage .create(result, snapshot, crypto.pureCrypto) .map(signedResult => OpenEnvelope(signedResult, Recipients.cc(participantId))) } - .toList - .sequence } yield Batch(envelopes) } diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/DomainNodeMediatorFactory.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/DomainNodeMediatorFactory.scala deleted file mode 100644 index f8174f940..000000000 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/DomainNodeMediatorFactory.scala +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.digitalasset.canton.domain.mediator - -import cats.data.EitherT -import com.digitalasset.canton.config.RequireTypes.InstanceName -import com.digitalasset.canton.crypto.{Crypto, KeyName, PublicKey, SigningPublicKey} -import com.digitalasset.canton.domain.topology.DomainTopologyManager -import com.digitalasset.canton.topology.{DomainId, MediatorId} -import com.digitalasset.canton.tracing.TraceContext - -import scala.concurrent.{ExecutionContext, Future} - -trait DomainNodeMediatorFactory { - - /** When the domain node first initializes it needs an initial mediator key to create the essential state of the domain. */ - def fetchInitialMediatorKey( - domainName: InstanceName, - domainId: DomainId, - mediatorId: MediatorId, - namespaceKey: PublicKey, - topologyManager: DomainTopologyManager, - crypto: Crypto, - )(implicit - ec: ExecutionContext, - traceContext: TraceContext, - ): EitherT[Future, String, SigningPublicKey] - - def mediatorRuntimeFactory: MediatorRuntimeFactory -} - -object DomainNodeMediatorFactory { - - /** Trait for extended in the enterprise environment */ - trait Embedded extends DomainNodeMediatorFactory { - override def fetchInitialMediatorKey( - domainName: InstanceName, - domainId: DomainId, - mediatorId: MediatorId, - namespaceKey: PublicKey, - topologyManager: DomainTopologyManager, - crypto: Crypto, - )(implicit - ec: ExecutionContext, - traceContext: TraceContext, - ): EitherT[Future, String, SigningPublicKey] = - for { - keyName <- EitherT.fromEither[Future](KeyName.create(s"$domainName-mediator-signing")) - res <- crypto - .generateSigningKey(name = Some(keyName)) - .leftMap(err => s"Failed to generate mediator signing key: $err") - } yield res - - override val mediatorRuntimeFactory: MediatorRuntimeFactory = CommunityMediatorRuntimeFactory - } - - /** The Mediator is operating within the DomainNode */ - object Embedded extends Embedded - -} diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/Mediator.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/Mediator.scala index adad21ac7..2fe8e91c0 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/Mediator.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/Mediator.scala @@ -32,6 +32,7 @@ import com.digitalasset.canton.topology.client.DomainTopologyClientWithInit import com.digitalasset.canton.topology.processing.TopologyTransactionProcessor import com.digitalasset.canton.tracing.{NoTracing, TraceContext, Traced} import com.digitalasset.canton.util.ShowUtil._ +import com.digitalasset.canton.version.ProtocolVersion import com.google.common.annotations.VisibleForTesting import io.opentelemetry.api.trace.Tracer @@ -51,6 +52,7 @@ class Mediator( sequencerCounterTrackerStore: SequencerCounterTrackerStore, sequencedEventStore: SequencedEventStore, parameters: LocalNodeParameters, + protocolVersion: ProtocolVersion, clock: Clock, metrics: MediatorMetrics, readyCheck: MediatorReadyCheck, @@ -80,6 +82,7 @@ class Mediator( timeTracker, state, new LoggingAlarmStreamer(logger), + protocolVersion, loggerFactory, ) diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/MediatorRuntimeFactory.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/MediatorRuntimeFactory.scala index d97d4a0aa..56d8c60cc 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/MediatorRuntimeFactory.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/mediator/MediatorRuntimeFactory.scala @@ -22,6 +22,7 @@ import com.digitalasset.canton.topology.{DomainId, MediatorId} import com.digitalasset.canton.topology.client.DomainTopologyClientWithInit import com.digitalasset.canton.topology.processing.TopologyTransactionProcessor import com.digitalasset.canton.tracing.TraceContext +import com.digitalasset.canton.version.ProtocolVersion import io.grpc.ServerServiceDefinition import io.opentelemetry.api.trace.Tracer @@ -73,6 +74,7 @@ trait MediatorRuntimeFactory { topologyTransactionProcessor: TopologyTransactionProcessor, timeTrackerConfig: DomainTimeTrackerConfig, nodeParameters: LocalNodeParameters, + protocolVersion: ProtocolVersion, clock: Clock, metrics: MediatorMetrics, futureSupervisor: FutureSupervisor, @@ -97,6 +99,7 @@ object CommunityMediatorRuntimeFactory extends MediatorRuntimeFactory { topologyTransactionProcessor: TopologyTransactionProcessor, timeTrackerConfig: DomainTimeTrackerConfig, nodeParameters: LocalNodeParameters, + protocolVersion: ProtocolVersion, clock: Clock, metrics: MediatorMetrics, futureSupervisor: FutureSupervisor, @@ -132,6 +135,7 @@ object CommunityMediatorRuntimeFactory extends MediatorRuntimeFactory { sequencerCounterTrackerStore, sequencedEventStore, nodeParameters, + protocolVersion, clock, metrics, MediatorReadyCheck(mediatorId, syncCrypto, loggerFactory), diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/sequencing/admin/SequencerInitialization.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/sequencing/admin/SequencerInitialization.scala index 21f537b3a..1e7476079 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/sequencing/admin/SequencerInitialization.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/sequencing/admin/SequencerInitialization.scala @@ -8,14 +8,13 @@ import com.digitalasset.canton.config.ProcessingTimeout import com.digitalasset.canton.config.RequireTypes.InstanceName import com.digitalasset.canton.crypto.store.CryptoPublicStore import com.digitalasset.canton.crypto.{KeyName, KeyPurpose, PublicKey} -import com.digitalasset.canton.domain.topology.DomainTopologyManager import com.digitalasset.canton.domain.sequencing.admin.client.SequencerAdminClient import com.digitalasset.canton.domain.sequencing.admin.protocol.InitRequest -import com.digitalasset.canton.topology.transaction.{OwnerToKeyMapping, TopologyStateUpdate} -import com.digitalasset.canton.topology.SequencerId import com.digitalasset.canton.logging.NamedLoggerFactory import com.digitalasset.canton.sequencing.client.SequencerClientConfig import com.digitalasset.canton.sequencing.handshake.SequencerHandshake +import com.digitalasset.canton.topology.SequencerId +import com.digitalasset.canton.topology.transaction.{OwnerToKeyMapping, TopologyStateUpdateMapping} import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.version.ProtocolVersion @@ -33,7 +32,7 @@ object SequencerInitialization { config: SequencerClientConfig, devVersionSupport: Boolean, sequencerAdminClient: SequencerAdminClient, - topologyManager: DomainTopologyManager, + authorizeStateUpdate: TopologyStateUpdateMapping => EitherT[Future, String, Unit], cryptoPublicStore: CryptoPublicStore, initRequest: InitRequest, timeouts: ProcessingTimeout, @@ -62,18 +61,15 @@ object SequencerInitialization { exists <- cryptoPublicStore .existsPublicKey(response.publicKey.fingerprint, KeyPurpose.Signing) .leftMap(_.toString) - _ = if (!exists) - cryptoPublicStore.storePublicKey( - response.publicKey, - Some(KeyName.tryCreate(s"$domainName-signing")), - ) - else () - _ <- topologyManager - .authorize( - TopologyStateUpdate.createAdd(mapping), - Some(initRequest.domainId.unwrap.namespace.fingerprint), - force = false, - ) - .leftMap(_.toString) + _ <- + if (!exists) + cryptoPublicStore + .storePublicKey( + response.publicKey, + Some(KeyName.tryCreate(s"$domainName-signing")), + ) + .leftMap(_.toString) + else EitherT.rightT[Future, String](()) + _ <- authorizeStateUpdate(mapping) } yield response.publicKey } diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationService.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationService.scala index cad8ff592..ddc2e20f8 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationService.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationService.scala @@ -92,7 +92,7 @@ class MemberAuthenticationService( .toRight(NoKeysRegistered(member): AuthenticationError) } ) - nonce = Nonce.generate() + nonce = Nonce.generate(cryptoApi.pureCrypto) storedNonce = StoredNonce(member, nonce, clock.now, nonceExpirationTime) _ <- EitherT.right(store.saveNonce(storedNonce)) } yield { @@ -142,7 +142,7 @@ class MemberAuthenticationService( _ <- agreementManager.fold(EitherT.rightT[Future, AuthenticationError](())) { manager => storeAcceptedAgreement(member, manager, manager.agreement.id, signature, generatedAt) } - token = AuthenticationToken.generate() + token = AuthenticationToken.generate(cryptoApi.pureCrypto) tokenExpiry = clock.now.add(tokenExpirationTime) storedToken = StoredAuthenticationToken(member, tokenExpiry, token) _ <- EitherT.right(tokenCache.saveToken(storedToken)) diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/topology/DomainTopologyManager.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/topology/DomainTopologyManager.scala index 822c43bce..cc139c784 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/topology/DomainTopologyManager.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/topology/DomainTopologyManager.scala @@ -198,7 +198,7 @@ class DomainTopologyManager( filterNamespace = None, ) ) - keys = txs.adds.toIdentityState + keys = txs.adds.toTopologyState .map(_.mapping) .collect { case OwnerToKeyMapping(`participantId`, key) => key diff --git a/community/domain/src/main/scala/com/digitalasset/canton/domain/topology/client/DomainInitializationObserver.scala b/community/domain/src/main/scala/com/digitalasset/canton/domain/topology/client/DomainInitializationObserver.scala index 8573c0119..5fc006b87 100644 --- a/community/domain/src/main/scala/com/digitalasset/canton/domain/topology/client/DomainInitializationObserver.scala +++ b/community/domain/src/main/scala/com/digitalasset/canton/domain/topology/client/DomainInitializationObserver.scala @@ -59,7 +59,7 @@ class DomainInitializationObserver( /** returns true if the initialisation data exists (but might not yet be effective) */ def initialisedAtHead: Future[Boolean] = sequencedStore.timestamp().flatMap { - case Some(ts) => initialisedAt(ts.immediateSuccessor) + case Some((_, effective)) => initialisedAt(effective.value.immediateSuccessor) case None => Future.successful(false) } diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/ConfirmationResponseProcessorTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/ConfirmationResponseProcessorTest.scala index 8f7fa1ee1..d0754480e 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/ConfirmationResponseProcessorTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/ConfirmationResponseProcessorTest.scala @@ -33,6 +33,7 @@ import com.digitalasset.canton.topology.client.TopologySnapshot import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.util.MonadUtil.{sequentialTraverse, sequentialTraverse_} import com.digitalasset.canton.util.ShowUtil._ +import com.digitalasset.canton.version.ProtocolVersion import com.google.protobuf.ByteString import org.mockito.ArgumentCaptor import org.mockito.ArgumentMatchers.{eq => eqMatch} @@ -112,6 +113,7 @@ class ConfirmationResponseProcessorTest extends AsyncWordSpec with BaseTest { timeTracker, mediatorState, new LoggingAlarmStreamer(logger), + ProtocolVersion.latestForTest, loggerFactory, ) @@ -183,6 +185,7 @@ class ConfirmationResponseProcessorTest extends AsyncWordSpec with BaseTest { Some(fullInformeeTree.transactionId.toRootHash), confirmers, factory.domainId, + ProtocolVersion.latestForTest, ) val participantCrypto = identityFactory.forOwner(participant) SignedProtocolMessage.tryCreate( @@ -840,6 +843,7 @@ class ConfirmationResponseProcessorTest extends AsyncWordSpec with BaseTest { Some(fullInformeeTree.transactionId.toRootHash), Set.empty, factory.domainId, + ProtocolVersion.latestForTest, ) val participantCrypto = identityFactory2.forOwner(participant) SignedProtocolMessage.tryCreate( diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/MediatorStateTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/MediatorStateTest.scala index 66eece533..9f9ee46d0 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/MediatorStateTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/MediatorStateTest.scala @@ -30,7 +30,7 @@ class MediatorStateTest extends AsyncWordSpec with BaseTest { val bob = ConfirmingParty(LfPartyId.assertFromString("bob"), 2) val hashOps: HashOps = new SymbolicPureCrypto val h: Int => Hash = TestHash.digest - val s: Int => Salt = TestSalt.generate + val s: Int => Salt = TestSalt.generateSalt def rh(index: Int): RootHash = RootHash(h(index)) val viewCommonData = ViewCommonData.create(hashOps)(Set(alice, bob), NonNegativeInt.tryCreate(2), s(999)) diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/ResponseAggregationTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/ResponseAggregationTest.scala index 9645c3d58..a47a657ca 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/ResponseAggregationTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/ResponseAggregationTest.scala @@ -26,6 +26,7 @@ import com.digitalasset.canton.topology.transaction.TrustLevel import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.{BaseTest, LfPartyId} import com.digitalasset.canton.util.ShowUtil._ +import com.digitalasset.canton.version.ProtocolVersion import org.scalatest.funspec.PathAnyFunSpec import scala.concurrent.{ExecutionContext, Future} @@ -38,7 +39,7 @@ class ResponseAggregationTest extends PathAnyFunSpec with BaseTest { def b[A](i: Int): BlindedNode[A] = BlindedNode(RootHash(TestHash.digest(i))) val hashOps: HashOps = new SymbolicPureCrypto - def salt(i: Int): Salt = TestSalt.generate(i) + def salt(i: Int): Salt = TestSalt.generateSalt(i) val domainId = DefaultTestIdentities.domainId val mediatorId = DefaultTestIdentities.mediator @@ -78,6 +79,7 @@ class ResponseAggregationTest extends PathAnyFunSpec with BaseTest { rootHashO, confirmingParties, domainId, + ProtocolVersion.latestForTest, ) describe("under the Signatory policy") { @@ -142,6 +144,7 @@ class ResponseAggregationTest extends PathAnyFunSpec with BaseTest { someOtherRootHash, Set(alice.party), domainId, + ProtocolVersion.latestForTest, ) val result = sut @@ -382,6 +385,7 @@ class ResponseAggregationTest extends PathAnyFunSpec with BaseTest { None, Set.empty, domainId, + ProtocolVersion.latestForTest, ) val result = loggerFactory.assertLogs( valueOrFail(sut.progress(changeTs, response, topologySnapshot).value.futureValue)( @@ -474,7 +478,16 @@ class ResponseAggregationTest extends PathAnyFunSpec with BaseTest { it("should ignore malformed non-VIP responses") { val reject = LocalReject.MalformedRejects.Payloads.Reject("malformed request") val response = - MediatorResponse.tryCreate(requestId, nonVip, None, reject, None, Set.empty, domainId) + MediatorResponse.tryCreate( + requestId, + nonVip, + None, + reject, + None, + Set.empty, + domainId, + ProtocolVersion.latestForTest, + ) val result = loggerFactory.assertLogs( valueOrFail( sut diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/store/FinalizedResponseStoreTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/store/FinalizedResponseStoreTest.scala index dcbaede1e..1be2ea2ba 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/store/FinalizedResponseStoreTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/mediator/store/FinalizedResponseStoreTest.scala @@ -38,7 +38,7 @@ trait FinalizedResponseStoreTest extends BeforeAndAfterAll { val hashOps = new SymbolicPureCrypto def h(i: Int): Hash = TestHash.digest(i) def rh(index: Int): RootHash = RootHash(h(index)) - def s(i: Int): Salt = TestSalt.generate(i) + def s(i: Int): Salt = TestSalt.generateSalt(i) val viewCommonData = ViewCommonData.create(hashOps)(Set(alice, bob), NonNegativeInt.tryCreate(2), s(999)) val view = TransactionView(hashOps)(viewCommonData, BlindedNode(rh(0)), Nil) diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/AuthenticationTokenCacheTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/AuthenticationTokenCacheTest.scala index 244780dec..d80fbbbb8 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/AuthenticationTokenCacheTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/AuthenticationTokenCacheTest.scala @@ -4,6 +4,7 @@ package com.digitalasset.canton.domain.sequencing.authentication import com.digitalasset.canton.BaseTest +import com.digitalasset.canton.crypto.provider.symbolic.SymbolicPureCrypto import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.topology.DefaultTestIdentities import com.digitalasset.canton.sequencing.authentication.AuthenticationToken @@ -19,10 +20,11 @@ class AuthenticationTokenCacheTest extends AsyncWordSpec with BaseTest { val cache = new AuthenticationTokenCache(clock, storeSpy, loggerFactory) } - val participant1 = DefaultTestIdentities.participant1 - val participant2 = DefaultTestIdentities.participant2 - val token1 = AuthenticationToken.generate() - val token2 = AuthenticationToken.generate() + lazy val participant1 = DefaultTestIdentities.participant1 + lazy val participant2 = DefaultTestIdentities.participant2 + lazy val crypto = new SymbolicPureCrypto + lazy val token1 = AuthenticationToken.generate(crypto) + lazy val token2 = AuthenticationToken.generate(crypto) def ts(n: Int): CantonTimestamp = CantonTimestamp.Epoch.plusSeconds(n.toLong) "lookup" should { diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationServiceTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationServiceTest.scala index 189f8193e..e9caaece2 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationServiceTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationServiceTest.scala @@ -89,7 +89,9 @@ class MemberAuthenticationServiceTest extends AsyncWordSpec with BaseTest { val sut = service(false) for { generateNonceError <- leftOrFail(sut.generateNonce(p1))("generating nonce") - validateSignatureError <- leftOrFail(sut.validateSignature(p1, null, Nonce.generate()))( + validateSignatureError <- leftOrFail( + sut.validateSignature(p1, null, Nonce.generate(syncCrypto.pureCrypto)) + )( "validateSignature" ) validateTokenError <- leftOrFail(sut.validateToken(domainId, p1, null))( diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationStoreTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationStoreTest.scala index 1f553735d..dc7642ab3 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationStoreTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/MemberAuthenticationStoreTest.scala @@ -6,6 +6,7 @@ package com.digitalasset.canton.domain.sequencing.authentication import cats.syntax.traverse._ import com.digitalasset.canton.BaseTest import com.digitalasset.canton.crypto.Nonce +import com.digitalasset.canton.crypto.provider.symbolic.SymbolicPureCrypto import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.topology.{DefaultTestIdentities, Member} import com.digitalasset.canton.resource.DbStorage @@ -17,10 +18,11 @@ import org.scalatest.wordspec.AsyncWordSpec import scala.concurrent.Future trait MemberAuthenticationStoreTest extends AsyncWordSpec with BaseTest { - val participant1 = DefaultTestIdentities.participant1 - val participant2 = DefaultTestIdentities.participant2 - val participant3 = DefaultTestIdentities.participant3 - val defaultExpiry = CantonTimestamp.Epoch.plusSeconds(120) + lazy val participant1 = DefaultTestIdentities.participant1 + lazy val participant2 = DefaultTestIdentities.participant2 + lazy val participant3 = DefaultTestIdentities.participant3 + lazy val defaultExpiry = CantonTimestamp.Epoch.plusSeconds(120) + lazy val crypto = new SymbolicPureCrypto def memberAuthenticationStore(mk: () => MemberAuthenticationStore): Unit = { "invalid member" should { @@ -129,9 +131,9 @@ trait MemberAuthenticationStoreTest extends AsyncWordSpec with BaseTest { member: Member, expiry: CantonTimestamp = defaultExpiry, ): StoredAuthenticationToken = - StoredAuthenticationToken(member, expiry, AuthenticationToken.generate()) + StoredAuthenticationToken(member, expiry, AuthenticationToken.generate(crypto)) def generateNonce(member: Member, expiry: CantonTimestamp = defaultExpiry): StoredNonce = - StoredNonce(member, Nonce.generate(), CantonTimestamp.Epoch, expiry) + StoredNonce(member, Nonce.generate(crypto), CantonTimestamp.Epoch, expiry) } class MemberAuthenticationStoreTestInMemory extends MemberAuthenticationStoreTest { diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/grpc/SequencerAuthenticationServerInterceptorTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/grpc/SequencerAuthenticationServerInterceptorTest.scala index 516d78132..64881024b 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/grpc/SequencerAuthenticationServerInterceptorTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/authentication/grpc/SequencerAuthenticationServerInterceptorTest.scala @@ -5,6 +5,7 @@ package com.digitalasset.canton.domain.sequencing.authentication.grpc import cats.data.EitherT import com.digitalasset.canton.config.DefaultProcessingTimeouts +import com.digitalasset.canton.crypto.provider.symbolic.SymbolicPureCrypto import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.domain.api.v0.{Hello, HelloServiceGrpc} import com.digitalasset.canton.domain.governance.ParticipantAuditor @@ -97,8 +98,10 @@ class SequencerAuthenticationServerInterceptorTest val unauthenticatedMemberId = UniqueIdentifier.fromProtoPrimitive_("unm1::default").map(new UnauthenticatedMemberId(_)).value val neverExpire = CantonTimestamp.MaxValue - val token = AuthenticationTokenWithExpiry(AuthenticationToken.generate(), neverExpire) - val incorrectToken = AuthenticationTokenWithExpiry(AuthenticationToken.generate(), neverExpire) + val crypto = new SymbolicPureCrypto + val token = AuthenticationTokenWithExpiry(AuthenticationToken.generate(crypto), neverExpire) + val incorrectToken = + AuthenticationTokenWithExpiry(AuthenticationToken.generate(crypto), neverExpire) require(token != incorrectToken, "The generated tokens must be different") diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/service/GrpcSequencerIntegrationTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/service/GrpcSequencerIntegrationTest.scala index df6bc027a..7a91a5289 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/service/GrpcSequencerIntegrationTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/service/GrpcSequencerIntegrationTest.scala @@ -116,7 +116,7 @@ case class Env(loggerFactory: NamedLoggerFactory)(implicit .Success( v0.Challenge.Success( ReleaseVersion.current.toProtoPrimitive, - Nonce.generate().toProtoPrimitive, + Nonce.generate(cryptoApi.pureCrypto).toProtoPrimitive, fingerprints.map(_.unwrap), ) ) @@ -129,7 +129,7 @@ case class Env(loggerFactory: NamedLoggerFactory)(implicit v0.Authentication.Response.Value .Success( v0.Authentication.Success( - AuthenticationToken.generate().toProtoPrimitive, + AuthenticationToken.generate(cryptoApi.pureCrypto).toProtoPrimitive, Some(clock.now.plusSeconds(100000).toProtoPrimitive), ) ) diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/service/GrpcSequencerServiceTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/service/GrpcSequencerServiceTest.scala index 393f313f3..289bca4fe 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/service/GrpcSequencerServiceTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/sequencing/service/GrpcSequencerServiceTest.scala @@ -10,6 +10,7 @@ import com.daml.nonempty.NonEmpty import com.digitalasset.canton.concurrent.Threading import com.digitalasset.canton.config.RequireTypes.NonNegativeInt import com.digitalasset.canton.crypto.DomainSyncCryptoClient +import com.digitalasset.canton.crypto.provider.symbolic.SymbolicPureCrypto import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.domain.api.v0 import com.digitalasset.canton.domain.governance.ParticipantAuditor @@ -73,8 +74,10 @@ class GrpcSequencerServiceTest extends FixtureAsyncWordSpec with BaseTest { override def close(): Unit = {} } - private val participant = DefaultTestIdentities.participant1 - private val unauthenticatedMember = UnauthenticatedMemberId.tryCreate(participant.uid.namespace) + private lazy val participant = DefaultTestIdentities.participant1 + private lazy val crypto = new SymbolicPureCrypto + private lazy val unauthenticatedMember = + UnauthenticatedMemberId.tryCreate(participant.uid.namespace)(crypto) class Environment(member: Member) extends Matchers { val sequencer: Sequencer = mock[Sequencer] diff --git a/community/domain/src/test/scala/com/digitalasset/canton/domain/topology/DomainTopologyManagerRequestServiceTest.scala b/community/domain/src/test/scala/com/digitalasset/canton/domain/topology/DomainTopologyManagerRequestServiceTest.scala index 71c96daa8..8ed198b70 100644 --- a/community/domain/src/test/scala/com/digitalasset/canton/domain/topology/DomainTopologyManagerRequestServiceTest.scala +++ b/community/domain/src/test/scala/com/digitalasset/canton/domain/topology/DomainTopologyManagerRequestServiceTest.scala @@ -266,7 +266,7 @@ class DomainTopologyManagerRequestServiceTest extends AsyncWordSpec with BaseTes when(manager.store).thenReturn(store) for { res <- service.newRequest(List(p1Mapping, p1MappingEnc, trustCert)) - dis <- store.headTransactions.map(_.toIdentityState) + dis <- store.headTransactions.map(_.toTopologyState) } yield { res.map(_.state) shouldBe Seq(Accepted, Accepted, Accepted) dis should have length (4) diff --git a/community/participant/src/main/daml/daml.yaml b/community/participant/src/main/daml/daml.yaml index 633c086ce..9294e7a95 100644 --- a/community/participant/src/main/daml/daml.yaml +++ b/community/participant/src/main/daml/daml.yaml @@ -1,4 +1,4 @@ -sdk-version: 2.2.0-snapshot.20220504.9851.0.4c8e027d +sdk-version: 2.3.0-snapshot.20220511.9887.0.5a509164 name: AdminWorkflows source: AdminWorkflows.daml version: 2.3.0 diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/ParticipantNode.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/ParticipantNode.scala index e0a299fe7..2397a6eca 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/ParticipantNode.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/ParticipantNode.scala @@ -19,7 +19,7 @@ import com.digitalasset.canton.concurrent.{ } import com.digitalasset.canton.config.RequireTypes.InstanceName import com.digitalasset.canton.config.{DbConfig, H2DbConfig, TestingConfigInternal} -import com.digitalasset.canton.crypto.{CryptoPureApi, KeyName, SyncCryptoApiProvider} +import com.digitalasset.canton.crypto.{CryptoPureApi, SyncCryptoApiProvider} import com.digitalasset.canton.domain.api.v0.DomainTimeServiceGrpc import com.digitalasset.canton.environment.{CantonNode, CantonNodeBootstrapBase} import com.digitalasset.canton.health.admin.data.ParticipantStatus @@ -121,7 +121,7 @@ class ParticipantNodeBootstrap( ) { /** per session created admin token for in-process connections to ledger-api */ - val adminToken: CantonAdminToken = CantonAdminToken.create() + val adminToken: CantonAdminToken = CantonAdminToken.create(crypto.pureCrypto) override protected def connectionPoolForParticipant: Boolean = true @@ -134,10 +134,11 @@ class ParticipantNodeBootstrap( None ) + private val authorizedTopologyStore = topologyStoreFactory.forId(AuthorizedStore) private val topologyManager = new ParticipantTopologyManager( clock, - topologyStoreFactory.forId(AuthorizedStore), + authorizedTopologyStore, crypto, cantonParameterConfig.processingTimeouts, loggerFactory, @@ -218,20 +219,11 @@ class ParticipantNodeBootstrap( withNewTraceContext { implicit traceContext => for { // create keys - nsName <- EitherT.fromEither[Future](KeyName.create(s"$name-namespace")) - namespaceKey <- crypto - .generateSigningKey(name = Some(nsName)) - .leftMap(err => s"Failed to generate key for namespace: $err") - sgName <- EitherT.fromEither[Future](KeyName.create(s"$name-signing")) - signingKey <- crypto - .generateSigningKey(name = Some(sgName)) - .leftMap(err => s"Failed to generate key for signing: $err") - encryptName <- EitherT.fromEither[Future](KeyName.create(s"$name-encryption")) - encryptionKey <- crypto - .generateEncryptionKey(name = Some(encryptName)) - .leftMap(err => s"Failed to generate key for encryption: $err") - - // init id + namespaceKey <- getOrCreateSigningKey(s"$name-namespace") + signingKey <- getOrCreateSigningKey(s"$name-signing") + encryptionKey <- getOrCreateEncryptionKey(s"$name-encryption") + + // create id identifier <- EitherT .fromEither[Future](Identifier.create(name.unwrap)) .leftMap(err => s"Failed to convert participant name to identifier: $err") @@ -240,7 +232,6 @@ class ParticipantNodeBootstrap( Namespace(namespaceKey.fingerprint), ) nodeId = NodeId(uid) - _ <- storeId(nodeId) // init topology manager participantId = ParticipantId(uid) @@ -266,10 +257,14 @@ class ParticipantNodeBootstrap( namespaceKey, OwnerToKeyMapping(participantId, encryptionKey), ) - // initialize certificate - _ <- new LegalIdentityInit(certificateGenerator, crypto) - .initializeCertificate(uid, Seq(participantId), namespaceKey)(topologyManager) + _ <- (new LegalIdentityInit(certificateGenerator, crypto)).checkOrInitializeCertificate( + uid, + Seq(participantId), + namespaceKey, + )(topologyManager, authorizedTopologyStore) + // finally, we store the node id, which means that the node will not be auto-initialised next time when we start + _ <- storeId(nodeId) } yield () } diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/admin/RepairService.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/admin/RepairService.scala index 7c06386d0..d3e082986 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/admin/RepairService.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/admin/RepairService.scala @@ -24,7 +24,7 @@ import com.digitalasset.canton.config.RequireTypes.{ LengthLimitedStringWrapperCompanion, String255, } -import com.digitalasset.canton.crypto.{HashPurpose, SecureRandomness, SyncCryptoApiProvider} +import com.digitalasset.canton.crypto.{HashPurpose, SyncCryptoApiProvider} import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.lifecycle.{AsyncOrSyncCloseable, FlagCloseableAsync, HasCloseContext} import com.digitalasset.canton.logging.pretty.{Pretty, PrettyPrinting} @@ -1035,7 +1035,7 @@ class RepairService( // We take as much entropy as for a random UUID. // This should be enough to guard against clashes between the repair requests executed on a single participant. // We don't have to worry about clashes with ordinary transaction IDs as the hash purpose is different. - val randomness = SecureRandomness.randomByteString(16) + val randomness = syncCrypto.pureCrypto.generateRandomByteString(16) val hash = syncCrypto.pureCrypto.digest(HashPurpose.RepairTransactionId, randomness) TransactionId(hash) } diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/config/LocalParticipantConfig.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/config/LocalParticipantConfig.scala index c6612a916..1a05d11d7 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/config/LocalParticipantConfig.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/config/LocalParticipantConfig.scala @@ -278,6 +278,8 @@ case class LedgerApiServerConfig( maxInboundMessageSize: NonNegativeInt = ServerConfig.defaultMaxInboundMessageSize, databaseConnectionTimeout: NonNegativeFiniteDuration = LedgerApiServerConfig.DefaultDatabaseConnectionTimeout, + apiStreamShutdownTimeout: NonNegativeFiniteDuration = + LedgerApiServerConfig.DefaultApiStreamShutdownTimeout, maxTransactionsInMemoryFanOutBufferSize: Long = LedgerApiServerConfig.DefaultMaxTransactionsInMemoryFanOutBufferSize, enableInMemoryFanOutForLedgerApi: Boolean = false, // Not tested for production yet @@ -310,6 +312,8 @@ object LedgerApiServerConfig { val DefaultDatabaseConnectionTimeout: NonNegativeFiniteDuration = NonNegativeFiniteDuration.ofSeconds(30) val DefaultMaxTransactionsInMemoryFanOutBufferSize: Long = 10000L + val DefaultApiStreamShutdownTimeout: NonNegativeFiniteDuration = + NonNegativeFiniteDuration.ofSeconds(5) // By default synchronous commit locally but don't await ACKs when replicated val DefaultSynchronousCommitMode: String = "LOCAL" @@ -345,6 +349,7 @@ object LedgerApiServerConfig { _maxContractKeyStateCacheSize, _maxTransactionsInMemoryFanOutBufferSize, _enableInMemoryFanOutForLedgerApi, + _apiStreamShutdownTimeout, ), _portFile, _seeding, @@ -531,9 +536,6 @@ case class IndexerConfig( ingestionParallelism: NonNegativeInt = NonNegativeInt.tryCreate(DamlIndexerConfig.DefaultIngestionParallelism), submissionBatchSize: Long = DamlIndexerConfig.DefaultSubmissionBatchSize, - tailingRateLimitPerSecond: NonNegativeInt = - NonNegativeInt.tryCreate(DamlIndexerConfig.DefaultTailingRateLimitPerSecond), - batchWithinMillis: Long = DamlIndexerConfig.DefaultBatchWithinMillis, enableCompression: Boolean = DamlIndexerConfig.DefaultEnableCompression, schemaMigrationAttempts: Int = IndexerStartupMode.DefaultSchemaMigrationAttempts, schemaMigrationAttemptBackoff: NonNegativeFiniteDuration = @@ -572,8 +574,6 @@ object IndexerConfig { batchingParallelism, ingestionParallelism, submissionBatchSize, - tailingRateLimitPerSecond, - batchWithinMillis, enableCompression, _haConfig, // consider making a subset of these settings configurable once the ha-dust settles postgresTcpKeepalivesIdle, @@ -603,8 +603,6 @@ object IndexerConfig { batchingParallelism = NonNegativeInt.tryCreate(batchingParallelism), ingestionParallelism = NonNegativeInt.tryCreate(ingestionParallelism), submissionBatchSize = submissionBatchSize, - tailingRateLimitPerSecond = NonNegativeInt.tryCreate(tailingRateLimitPerSecond), - batchWithinMillis = batchWithinMillis, enableCompression = enableCompression, schemaMigrationAttempts = schemaMigrationAttempts, schemaMigrationAttemptBackoff = diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/domain/DomainRegistryHelpers.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/domain/DomainRegistryHelpers.scala index 4ee698c1d..72d5833d7 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/domain/DomainRegistryHelpers.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/domain/DomainRegistryHelpers.scala @@ -239,13 +239,11 @@ trait DomainRegistryHelpers extends FlagCloseable with NamedLogging { domainId, config.domain, participantId, - nodeId, clock, config.timeTracker, participantNodeParameters.processingTimeouts, authorizedStore, targetDomainStore, - topologyClient, loggerFactory, sequencerClientFactory, cryptoApiProvider.crypto.pureCrypto, diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/domain/grpc/ParticipantInitializeTopology.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/domain/grpc/ParticipantInitializeTopology.scala index 52051445b..a063c685e 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/domain/grpc/ParticipantInitializeTopology.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/domain/grpc/ParticipantInitializeTopology.scala @@ -5,13 +5,15 @@ package com.digitalasset.canton.participant.domain.grpc import akka.stream.Materializer import cats.data.EitherT +import com.digitalasset.canton.DomainAlias import com.digitalasset.canton.config.ProcessingTimeout -import com.digitalasset.canton.crypto.HashOps +import com.digitalasset.canton.crypto.{HashOps, RandomOps} import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.lifecycle.FutureUnlessShutdown import com.digitalasset.canton.logging.{ErrorLoggingContext, NamedLoggerFactory} import com.digitalasset.canton.participant.domain.DomainRegistryError import com.digitalasset.canton.participant.topology.DomainOnboardingOutbox +import com.digitalasset.canton.protocol.messages.DefaultOpenEnvelope import com.digitalasset.canton.sequencing.client.{SequencerClient, SequencerClientFactory} import com.digitalasset.canton.sequencing.handlers.{ DiscardIgnoredEvents, @@ -19,22 +21,13 @@ import com.digitalasset.canton.sequencing.handlers.{ StripSignature, } import com.digitalasset.canton.sequencing.protocol.{Batch, Deliver} -import com.digitalasset.canton.sequencing.{ - BoxedEnvelope, - HandlerResult, - ResubscriptionStart, - UnsignedEnvelopeBox, - UnsignedProtocolEventHandler, -} +import com.digitalasset.canton.sequencing._ import com.digitalasset.canton.store.memory.{InMemorySendTrackerStore, InMemorySequencedEventStore} import com.digitalasset.canton.time.{Clock, DomainTimeTracker, DomainTimeTrackerConfig} -import com.digitalasset.canton.topology.client.DomainTopologyClientWithInit import com.digitalasset.canton.topology.store.TopologyStore -import com.digitalasset.canton.topology.{DomainId, NodeId, ParticipantId, UnauthenticatedMemberId} +import com.digitalasset.canton.topology.{DomainId, ParticipantId, UnauthenticatedMemberId} import com.digitalasset.canton.tracing.{TraceContext, Traced} import com.digitalasset.canton.util.MonadUtil -import com.digitalasset.canton.DomainAlias -import com.digitalasset.canton.protocol.messages.DefaultOpenEnvelope import io.opentelemetry.api.trace.Tracer import scala.concurrent.ExecutionContextExecutor @@ -50,16 +43,14 @@ object ParticipantInitializeTopology { domainId: DomainId, alias: DomainAlias, participantId: ParticipantId, - nodeId: NodeId, clock: Clock, timeTracker: DomainTimeTrackerConfig, processingTimeout: ProcessingTimeout, authorizedStore: TopologyStore, targetDomainStore: TopologyStore, - topologyClient: DomainTopologyClientWithInit, loggerFactory: NamedLoggerFactory, sequencerClientFactory: SequencerClientFactory, - hashOps: HashOps, + cryptoOps: HashOps with RandomOps, )(implicit executionContext: ExecutionContextExecutor, materializer: Materializer, @@ -67,7 +58,8 @@ object ParticipantInitializeTopology { traceContext: TraceContext, loggingContext: ErrorLoggingContext, ): EitherT[FutureUnlessShutdown, DomainRegistryError, Unit] = { - val unauthenticatedMember = UnauthenticatedMemberId.tryCreate(participantId.uid.namespace) + val unauthenticatedMember = + UnauthenticatedMemberId.tryCreate(participantId.uid.namespace)(cryptoOps) loggingContext.logger.debug( s"Unauthenticated member $unauthenticatedMember will register initial topology transactions on behalf of participant $participantId" @@ -108,7 +100,7 @@ object ParticipantInitializeTopology { FutureUnlessShutdown.outcomeF( client.subscribeAfterUnauthenticated( CantonTimestamp.MinValue, - DiscardIgnoredEvents { StripSignature { EnvelopeOpener(hashOps)(eventHandler) } }, + DiscardIgnoredEvents { StripSignature { EnvelopeOpener(cryptoOps)(eventHandler) } }, domainTimeTracker, ) ) diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/ledger/api/CantonAdminTokenAuthService.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/ledger/api/CantonAdminTokenAuthService.scala index 269e62c7a..50290cc49 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/ledger/api/CantonAdminTokenAuthService.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/ledger/api/CantonAdminTokenAuthService.scala @@ -3,19 +3,20 @@ package com.digitalasset.canton.participant.ledger.api -import java.util.concurrent.{CompletableFuture, CompletionStage} import com.daml.ledger.api.auth.{AuthService, ClaimSet} -import com.digitalasset.canton.crypto.SecureRandomness +import com.digitalasset.canton.crypto.RandomOps import com.digitalasset.canton.util.{HexString, NoCopy} import io.grpc.Metadata +import java.util.concurrent.{CompletableFuture, CompletionStage} + case class CantonAdminToken private (secret: String) extends NoCopy object CantonAdminToken { private[this] def apply(secret: String): CantonAdminToken = throw new UnsupportedOperationException("Use the create method instead.") - def create(): CantonAdminToken = { - val secret = HexString.toHexString(SecureRandomness.randomByteString(64)) + def create(randomOps: RandomOps): CantonAdminToken = { + val secret = HexString.toHexString(randomOps.generateRandomByteString(64)) new CantonAdminToken(secret) } } diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/ledger/api/CantonLedgerApiServerWrapper.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/ledger/api/CantonLedgerApiServerWrapper.scala index 851d9b489..f1a046d8c 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/ledger/api/CantonLedgerApiServerWrapper.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/ledger/api/CantonLedgerApiServerWrapper.scala @@ -9,6 +9,7 @@ import cats.implicits._ import com.codahale.metrics.SharedMetricRegistries import com.daml.api.util.TimeProvider import com.daml.caching +import scala.jdk.DurationConverters._ import com.daml.ledger.api.health.HealthChecks import com.daml.ledger.api.v1.experimental_features.{ CommandDeduplicationFeatures, @@ -167,8 +168,6 @@ object CantonLedgerApiServerWrapper extends NoTracing { batchingParallelism = config.indexerConfig.batchingParallelism.unwrap, ingestionParallelism = config.indexerConfig.ingestionParallelism.unwrap, submissionBatchSize = config.indexerConfig.submissionBatchSize, - tailingRateLimitPerSecond = config.indexerConfig.tailingRateLimitPerSecond.unwrap, - batchWithinMillis = config.indexerConfig.batchWithinMillis, enableCompression = config.indexerConfig.enableCompression, haConfig = config.indexerLockIds.fold(HaConfig()) { case IndexerLockIds(mainLockId, workerLockId) => @@ -363,6 +362,8 @@ object CantonLedgerApiServerWrapper extends NoTracing { ), ), userManagementConfig = config.serverConfig.userManagementService.damlConfig, + apiStreamShutdownTimeout = + config.serverConfig.apiStreamShutdownTimeout.duration.toScala, ).acquire() } yield () diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/BadRootHashMessagesRequestProcessor.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/BadRootHashMessagesRequestProcessor.scala index 950517a2e..48efe27de 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/BadRootHashMessagesRequestProcessor.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/BadRootHashMessagesRequestProcessor.scala @@ -18,6 +18,7 @@ import com.digitalasset.canton.sequencing.protocol.Recipients import com.digitalasset.canton.topology.{MediatorId, ParticipantId} import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.util.ShowUtil._ +import com.digitalasset.canton.version.ProtocolVersion import io.functionmeta.functionFullName import scala.concurrent.ExecutionContext @@ -27,6 +28,7 @@ class BadRootHashMessagesRequestProcessor( crypto: DomainSyncCryptoClient, sequencerClient: SequencerClient, participantId: ParticipantId, + protocolVersion: ProtocolVersion, override protected val timeouts: ProcessingTimeout, override protected val loggerFactory: NamedLoggerFactory, )(implicit ec: ExecutionContext) @@ -84,6 +86,7 @@ class BadRootHashMessagesRequestProcessor( rootHash = Some(rootHash), confirmingParties = Set.empty, domainId = domainId, + protocolVersion = protocolVersion, ) ) signedRejection <- signResponse(snapshot, rejection) diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/TransactionProcessingSteps.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/TransactionProcessingSteps.scala index ea0f06085..ad8b31ce6 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/TransactionProcessingSteps.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/TransactionProcessingSteps.scala @@ -18,6 +18,7 @@ import com.digitalasset.canton.crypto.{ DomainSyncCryptoClient, HashOps, HkdfInfo, + ProtocolCryptoApi, SecureRandomness, } import com.digitalasset.canton.data.ViewPosition.ListIndex @@ -494,7 +495,13 @@ class TransactionProcessingSteps( vt: TransactionViewMessage, optRandomness: Option[SecureRandomness], ): EitherT[Future, DecryptionError, LightTransactionViewTree] = - EncryptedViewMessage.decryptFor(snapshot, vt, participantId, optRandomness)( + EncryptedViewMessage.decryptFor( + snapshot, + vt, + participantId, + staticDomainParameters.protocolVersion, + optRandomness, + )( lightTransactionViewTreeDeserializer ) @@ -540,9 +547,14 @@ class TransactionProcessingSteps( val (subviewHash, index) = subviewHashAndIndex val info = HkdfInfo.subview(ListIndex(index)) for { - subviewRandomness <- pureCrypto - .hkdfExpand(randomness, randomness.unwrap.size, info) - .leftMap(error => EncryptedViewMessage.HkdfExpansionError(error, viewMessage)) + subviewRandomness <- + ProtocolCryptoApi + .hkdf(pureCrypto, staticDomainParameters.protocolVersion)( + randomness, + randomness.unwrap.size, + info, + ) + .leftMap(error => EncryptedViewMessage.HkdfExpansionError(error, viewMessage)) } yield { randomnessMap.get(subviewHash) match { case Some(promise) => diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/TransactionProcessor.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/TransactionProcessor.scala index fa1faf716..ab3b0c0df 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/TransactionProcessor.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/TransactionProcessor.scala @@ -73,7 +73,12 @@ class TransactionProcessor( domainId, participantId, confirmationRequestFactory, - new ConfirmationResponseFactory(participantId, domainId, loggerFactory), + new ConfirmationResponseFactory( + participantId, + domainId, + staticDomainParameters.protocolVersion, + loggerFactory, + ), ModelConformanceChecker( damle, confirmationRequestFactory.transactionTreeFactory, diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/ConfirmationRequestFactory.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/ConfirmationRequestFactory.scala index e07bbaba3..2fbbab272 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/ConfirmationRequestFactory.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/ConfirmationRequestFactory.scala @@ -74,16 +74,14 @@ class ConfirmationRequestFactory( val randomnessLength = EncryptedViewMessage.computeRandomnessLength(cryptoSnapshot) val keySeed = - optKeySeed.getOrElse(SecureRandomness.secureRandomness(randomnessLength)) + optKeySeed.getOrElse(cryptoSnapshot.pureCrypto.generateSecureRandomness(randomnessLength)) for { _ <- assertSubmittersNodeAuthorization(submitterInfo.actAs, cryptoSnapshot.ipsSnapshot) // Starting with Daml 1.6.0, the daml engine performs authorization validation. - transactionSeed <- seedGenerator - .generateSeedForTransaction(submitterInfo.changeId, domain, ledgerTime, transactionUuid) - .leftMap(SeedGeneratorError) + transactionSeed = seedGenerator.generateSaltSeed() transactionTree <- transactionTreeFactory .createTransactionTree( @@ -159,7 +157,7 @@ class ConfirmationRequestFactory( transactionTree: GenTransactionTree, cryptoSnapshot: DomainSnapshotSyncCryptoApi, keySeed: SecureRandomness, - version: ProtocolVersion, + protocolVersion: ProtocolVersion, )(implicit traceContext: TraceContext): EitherT[Future, ConfirmationRequestCreationError, List[ OpenEnvelope[TransactionViewMessage] ]] = { @@ -167,14 +165,14 @@ class ConfirmationRequestFactory( for { lightTreesWithMetadata <- EitherT.fromEither[Future]( transactionTree - .allLightTransactionViewTreesWithWitnessesAndSeeds(keySeed, pureCrypto) + .allLightTransactionViewTreesWithWitnessesAndSeeds(keySeed, pureCrypto, protocolVersion) .leftMap(KeySeedError) ) res <- lightTreesWithMetadata.toList .traverse { case (vt, witnesses, seed) => for { viewMessage <- EncryptedViewMessageFactory - .create(TransactionViewType)(vt, cryptoSnapshot, version, Some(seed)) + .create(TransactionViewType)(vt, cryptoSnapshot, protocolVersion, Some(seed)) .leftMap(EncryptedViewMessageCreationError) recipients <- witnesses .toRecipients(cryptoSnapshot.ipsSnapshot) diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/EncryptedViewMessageFactory.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/EncryptedViewMessageFactory.scala index b19dc11a4..5e592ad3e 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/EncryptedViewMessageFactory.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/EncryptedViewMessageFactory.scala @@ -22,7 +22,7 @@ object EncryptedViewMessageFactory { def create[VT <: ViewType](viewType: VT)( viewTree: viewType.View, cryptoSnapshot: DomainSnapshotSyncCryptoApi, - version: ProtocolVersion, + protocolVersion: ProtocolVersion, optRandomness: Option[SecureRandomness] = None, )(implicit traceContext: TraceContext, @@ -34,7 +34,7 @@ object EncryptedViewMessageFactory { val keyLength = EncryptedViewMessage.computeKeyLength(cryptoSnapshot) val randomnessLength = EncryptedViewMessage.computeRandomnessLength(cryptoSnapshot) val randomness: SecureRandomness = - optRandomness.getOrElse(SecureRandomness.secureRandomness(randomnessLength)) + optRandomness.getOrElse(cryptoPureApi.generateSecureRandomness(randomnessLength)) val informeeParties = viewTree.informees.map(_.party).toList def eitherT[B]( @@ -44,17 +44,24 @@ object EncryptedViewMessageFactory { for { symmetricViewKey <- eitherT( - cryptoPureApi.hkdfExpand(randomness, keyLength, HkdfInfo.ViewKey).leftMap(FailedToExpandKey) + ProtocolCryptoApi + .hkdf(cryptoPureApi, protocolVersion)(randomness, keyLength, HkdfInfo.ViewKey) + .leftMap(FailedToExpandKey) ) informeeParticipants <- cryptoSnapshot.ipsSnapshot .activeParticipantsOfAll(informeeParties) .leftMap(UnableToDetermineParticipant(_, cryptoSnapshot.domainId)) - keyMap <- createKeyMap(informeeParticipants.to(LazyList), randomness, cryptoSnapshot, version) + keyMap <- createKeyMap( + informeeParticipants.to(LazyList), + randomness, + cryptoSnapshot, + protocolVersion, + ) signature <- viewTree.toBeSigned .traverse(rootHash => cryptoSnapshot.sign(rootHash.unwrap).leftMap(FailedToSignViewMessage)) encryptedView <- eitherT( EncryptedView - .compressed[VT](cryptoPureApi, symmetricViewKey, viewType, version)(viewTree) + .compressed[VT](cryptoPureApi, symmetricViewKey, viewType, protocolVersion)(viewTree) .leftMap(FailedToEncryptViewMessage) ) } yield EncryptedViewMessage[VT]( diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/SeedGenerator.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/SeedGenerator.scala index 33c43ec5e..7f056f166 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/SeedGenerator.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/SeedGenerator.scala @@ -3,126 +3,15 @@ package com.digitalasset.canton.participant.protocol.submission -import java.util.UUID -import cats.data.EitherT -import com.daml.ledger.participant.state.v2.ChangeId import com.digitalasset.canton.crypto._ -import com.digitalasset.canton.participant.protocol.submission.SeedGenerator.{ - SeedData, - SeedForTransaction, - SeedForTransfer, -} -import com.digitalasset.canton.participant.protocol.transfer.TransferOutRequest -import com.digitalasset.canton.protocol.LfContractId -import com.digitalasset.canton.protocol.messages.DeliveredTransferOutResult -import com.digitalasset.canton.data.CantonTimestamp -import com.digitalasset.canton.serialization.DeterministicEncoding -import com.digitalasset.canton.topology.DomainId -import com.google.common.annotations.VisibleForTesting -import com.google.protobuf.ByteString - -import scala.concurrent.{ExecutionContext, Future} -/** Creates seeds and UUIDs for requests. - */ -class SeedGenerator(val hmacOps: HmacPrivateOps, hashOps: HashOps)(implicit ec: ExecutionContext) { - - /** Yields a hash from the method parameters and [[com.digitalasset.canton.crypto.HmacPrivateOps.hmac]]. - * It is assumed that the hash uniquely identifies the method parameters. - * - * If two instances of this class have different `hmacOps`, they will create different hashes, even if - * the method is called with the same parameters. - * - * Moreover, if the secret key used by `hmacOps` is unknown, the result of this method cannot be predicted. - */ - def generateSeedForTransaction( - changeId: ChangeId, - originDomain: DomainId, - let: CantonTimestamp, - transactionUuid: UUID, - ): EitherT[Future, SaltError, Salt] = { - val seedData = SeedForTransaction(changeId, originDomain, let, transactionUuid) - createSalt(seedData) - } - - def generateSeedForTransferOut( - request: TransferOutRequest, - transferOutUuid: UUID, - ): EitherT[Future, SaltError, Salt] = { - val seedData = SeedForTransfer( - request.contractId, - request.originDomain, - request.targetDomain, - hashOps.digest( - HashPurpose.TransferViewTreeMessageSeed, - request.targetTimeProof.getCryptographicEvidence, - ), - transferOutUuid, - ) - createSalt(seedData) - } +import java.util.UUID - def generateSeedForTransferIn( - contractId: LfContractId, - transferOutResultEvent: DeliveredTransferOutResult, - targetDomain: DomainId, - transferInUuid: UUID, - ): EitherT[Future, SaltError, Salt] = { - val seedData = SeedForTransfer( - contractId, - transferOutResultEvent.unwrap.domainId, - targetDomain, - hashOps.digest( - HashPurpose.TransferViewTreeMessageSeed, - transferOutResultEvent.result.content.getCryptographicEvidence, - ), - transferInUuid, - ) - createSalt(seedData) - } +/** Creates seeds and UUIDs for requests. */ +class SeedGenerator(randomOps: RandomOps) { def generateUuid(): UUID = UUID.randomUUID() - private def createSalt(seedData: SeedData): EitherT[Future, SaltError, Salt] = - Salt.generate(seedData.toDeterministicByteString, hmacOps) -} - -object SeedGenerator { - - trait SeedData { - def toDeterministicByteString: ByteString - } - - @VisibleForTesting - case class SeedForTransaction( - changeId: ChangeId, - domainId: DomainId, - let: CantonTimestamp, - transactionUuid: UUID, - ) extends SeedData { - override def toDeterministicByteString: ByteString = { - DeterministicEncoding - .encodeBytes(changeId.hash.bytes.toByteString) - .concat(DeterministicEncoding.encodeString(domainId.toProtoPrimitive)) - .concat(DeterministicEncoding.encodeInstant(let.toInstant)) - .concat(DeterministicEncoding.encodeString(transactionUuid.toString)) - } - } - - case class SeedForTransfer( - contractId: LfContractId, - originDomain: DomainId, - targetDomain: DomainId, - messageHash: Hash, - requestUuid: UUID, - ) extends SeedData { - override def toDeterministicByteString: ByteString = - DeterministicEncoding - .encodeString(contractId.toString) - .concat(DeterministicEncoding.encodeString(originDomain.toProtoPrimitive)) - .concat(DeterministicEncoding.encodeString(targetDomain.toProtoPrimitive)) - .concat(DeterministicEncoding.encodeBytes(messageHash.getCryptographicEvidence)) - .concat(DeterministicEncoding.encodeString(requestUuid.toString)) - } - + def generateSaltSeed(length: Int = SaltSeed.defaultLength): SaltSeed = + SaltSeed.generate(length)(randomOps) } diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/TransactionTreeFactory.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/TransactionTreeFactory.scala index b4dff196b..4de79bb66 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/TransactionTreeFactory.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/TransactionTreeFactory.scala @@ -6,7 +6,7 @@ package com.digitalasset.canton.participant.protocol.submission import cats.data.EitherT import com.daml.ledger.participant.state.v2.SubmitterInfo import com.digitalasset.canton._ -import com.digitalasset.canton.crypto.Salt +import com.digitalasset.canton.crypto.{Salt, SaltSeed} import com.digitalasset.canton.data.{ GenTransactionTree, TransactionView, @@ -40,7 +40,7 @@ trait TransactionTreeFactory { confirmationPolicy: ConfirmationPolicy, workflowId: Option[WorkflowId], mediatorId: MediatorId, - transactionSeed: Salt, + transactionSeed: SaltSeed, transactionUuid: UUID, topologySnapshot: TopologySnapshot, contractOfId: SerializableContractOfId, diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/TransactionTreeFactoryImpl.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/TransactionTreeFactoryImpl.scala index bd48e8db8..d517c7bdf 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/TransactionTreeFactoryImpl.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/submission/TransactionTreeFactoryImpl.scala @@ -14,7 +14,7 @@ import com.daml.lf.CantonOnly import com.daml.lf.data.Ref.PackageId import com.daml.lf.value.{Value, ValueCoder} import com.digitalasset.canton._ -import com.digitalasset.canton.crypto.{HashOps, HmacOps, Salt} +import com.digitalasset.canton.crypto.{HashOps, HmacOps, Salt, SaltSeed} import com.digitalasset.canton.data.ViewPosition.ListIndex import com.digitalasset.canton.data._ import com.digitalasset.canton.topology.{DomainId, MediatorId, ParticipantId} @@ -103,7 +103,7 @@ class TransactionTreeFactoryImpl( private[this] object State { def submission( - transactionSeed: Salt, + transactionSeed: SaltSeed, mediatorId: MediatorId, transactionUUID: UUID, ledgerTime: CantonTimestamp, @@ -131,7 +131,7 @@ class TransactionTreeFactoryImpl( confirmationPolicy: ConfirmationPolicy, workflowId: Option[WorkflowId], mediatorId: MediatorId, - transactionSeed: Salt, + transactionSeed: SaltSeed, transactionUuid: UUID, topologySnapshot: TopologySnapshot, contractOfId: SerializableContractOfId, diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferInProcessingSteps.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferInProcessingSteps.scala index 73f270d75..2bc58958c 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferInProcessingSteps.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferInProcessingSteps.scala @@ -72,7 +72,7 @@ class TransferInProcessingSteps( transferCoordination: TransferCoordination, seedGenerator: SeedGenerator, causalityTracking: Boolean, - version: ProtocolVersion, + protocolVersion: ProtocolVersion, protected val loggerFactory: NamedLoggerFactory, )(implicit val ec: ExecutionContext) extends TransferProcessingSteps[ @@ -161,14 +161,7 @@ class TransferInProcessingSteps( ) transferInUuid = seedGenerator.generateUuid() - seed <- seedGenerator - .generateSeedForTransferIn( - transferData.contract.contractId, - transferOutResult, - targetDomain, - transferInUuid, - ) - .leftMap(SeedGeneratorError) + seed = seedGenerator.generateSaltSeed() fullTree = makeFullTransferInTree( pureCrypto, seed, @@ -196,7 +189,7 @@ class TransferInProcessingSteps( ) viewMessage <- EncryptedViewMessageFactory - .create(TransferInViewType)(fullTree, recentSnapshot, version) + .create(TransferInViewType)(fullTree, recentSnapshot, protocolVersion) .leftMap[TransferProcessorError](EncryptionError) } yield { val rootHashMessage = @@ -251,7 +244,7 @@ class TransferInProcessingSteps( FullTransferInTree ]] = EncryptedViewMessage - .decryptFor(snapshot, envelope.protocolMessage, participantId) { bytes => + .decryptFor(snapshot, envelope.protocolMessage, participantId, protocolVersion) { bytes => FullTransferInTree .fromByteString(snapshot.pureCrypto)(bytes) .leftMap(e => DeserializationError(e.toString, bytes)) @@ -504,6 +497,7 @@ class TransferInProcessingSteps( txInRequest.toBeSigned, validationResult.confirmingParties, domainId, + protocolVersion, ) .leftMap(e => FailedToCreateResponse(e): TransferProcessorError) ) @@ -807,7 +801,7 @@ object TransferInProcessingSteps { def makeFullTransferInTree( pureCrypto: CryptoPureApi, - seed: Salt, + seed: SaltSeed, submitter: LfPartyId, stakeholders: Set[LfPartyId], contract: SerializableContract, diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutProcessingSteps.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutProcessingSteps.scala index b8e6d961a..85b8cc6d0 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutProcessingSteps.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutProcessingSteps.scala @@ -82,7 +82,7 @@ class TransferOutProcessingSteps( val engine: DAMLe, transferCoordination: TransferCoordination, seedGenerator: SeedGenerator, - version: ProtocolVersion, + protocolVersion: ProtocolVersion, protected val loggerFactory: NamedLoggerFactory, )(implicit val ec: ExecutionContext) extends TransferProcessingSteps[ @@ -159,10 +159,7 @@ class TransferOutProcessingSteps( (transferOutRequest, recipients) = transferOutRequestAndRecipients transferOutUuid = seedGenerator.generateUuid() - seed <- seedGenerator - .generateSeedForTransferOut(transferOutRequest, transferOutUuid) - .leftMap(SeedGeneratorError) - .mapK(FutureUnlessShutdown.outcomeK) + seed = seedGenerator.generateSaltSeed() fullTree = transferOutRequest.toFullTransferOutTree( pureCrypto, pureCrypto, @@ -172,7 +169,7 @@ class TransferOutProcessingSteps( mediatorMessage = fullTree.mediatorMessage rootHash = fullTree.rootHash viewMessage <- EncryptedViewMessageFactory - .create(TransferOutViewType)(fullTree, originRecentSnapshot, version) + .create(TransferOutViewType)(fullTree, originRecentSnapshot, protocolVersion) .leftMap[TransferProcessorError](EncryptionError) .mapK(FutureUnlessShutdown.outcomeK) maybeRecipients = Recipients.ofSet(recipients) @@ -244,10 +241,11 @@ class TransferOutProcessingSteps( FullTransferOutTree ]] = { EncryptedViewMessage - .decryptFor(originSnapshot, envelope.protocolMessage, participantId) { bytes => - FullTransferOutTree - .fromByteString(originSnapshot.pureCrypto)(bytes) - .leftMap(e => DeserializationError(e.toString, bytes)) + .decryptFor(originSnapshot, envelope.protocolMessage, participantId, protocolVersion) { + bytes => + FullTransferOutTree + .fromByteString(originSnapshot.pureCrypto)(bytes) + .leftMap(e => DeserializationError(e.toString, bytes)) } .map(WithRecipients(_, envelope.recipients)) } @@ -674,6 +672,7 @@ class TransferOutProcessingSteps( Some(rootHash), confirmingParties, domainId, + protocolVersion, ) ) Some(response) diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutRequest.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutRequest.scala index 260088e28..4e9bbb7b9 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutRequest.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutRequest.scala @@ -3,7 +3,7 @@ package com.digitalasset.canton.participant.protocol.transfer -import com.digitalasset.canton.crypto.{HashOps, HmacOps, Salt} +import com.digitalasset.canton.crypto.{HashOps, HmacOps, Salt, SaltSeed} import com.digitalasset.canton.data._ import com.digitalasset.canton.protocol.LfContractId import com.digitalasset.canton.time.TimeProof @@ -33,7 +33,7 @@ case class TransferOutRequest( def toFullTransferOutTree( hashOps: HashOps, hmacOps: HmacOps, - seed: Salt, + seed: SaltSeed, uuid: UUID, ): FullTransferOutTree = { val commonDataSalt = Salt.tryDeriveSalt(seed, 0, hmacOps) diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/validation/ConfirmationResponseFactory.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/validation/ConfirmationResponseFactory.scala index 240ccecff..c726bb605 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/validation/ConfirmationResponseFactory.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/protocol/validation/ConfirmationResponseFactory.scala @@ -16,6 +16,7 @@ import com.digitalasset.canton.protocol.{ConfirmationPolicy, LfContractId, Reque import com.digitalasset.canton.topology.{DomainId, ParticipantId} import com.digitalasset.canton.topology.client.TopologySnapshot import com.digitalasset.canton.tracing.TraceContext +import com.digitalasset.canton.version.ProtocolVersion import com.digitalasset.canton.{LfPartyId, checked} import scala.collection.mutable @@ -24,6 +25,7 @@ import scala.concurrent.{ExecutionContext, Future} class ConfirmationResponseFactory( participantId: ParticipantId, domainId: DomainId, + protocolVersion: ProtocolVersion, protected val loggerFactory: NamedLoggerFactory, ) extends NamedLogging { @@ -56,6 +58,7 @@ class ConfirmationResponseFactory( None, Set.empty, domainId, + protocolVersion, ) ) } @@ -276,6 +279,7 @@ class ConfirmationResponseFactory( rootHash, parties, domainId, + protocolVersion, ) ) } diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/pruning/AcsCommitmentProcessor.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/pruning/AcsCommitmentProcessor.scala index e4b124e50..0c9f676fd 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/pruning/AcsCommitmentProcessor.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/pruning/AcsCommitmentProcessor.scala @@ -42,6 +42,7 @@ import com.digitalasset.canton.util.EitherUtil.RichEither import com.digitalasset.canton.util._ import com.digitalasset.canton.util.retry.Policy import com.digitalasset.canton.LfPartyId +import com.digitalasset.canton.version.ProtocolVersion import com.google.common.annotations.VisibleForTesting import io.functionmeta.functionFullName @@ -134,6 +135,7 @@ class AcsCommitmentProcessor( commitmentPeriodObserver: (ExecutionContext, TraceContext) => FutureUnlessShutdown[Unit], killSwitch: => Unit, metrics: PruningMetrics, + protocolVersion: ProtocolVersion, override protected val timeouts: ProcessingTimeout, protected val loggerFactory: NamedLoggerFactory, )(implicit ec: ExecutionContext) @@ -642,7 +644,14 @@ class AcsCommitmentProcessor( cmt: AcsCommitment.CommitmentType, period: CommitmentPeriod, )(implicit traceContext: TraceContext): Future[SignedProtocolMessage[AcsCommitment]] = { - val payload = AcsCommitment.create(domainId, participantId, counterParticipant, period, cmt) + val payload = AcsCommitment.create( + domainId, + participantId, + counterParticipant, + period, + cmt, + protocolVersion, + ) SignedProtocolMessage.tryCreate(payload, crypto, domainCrypto.pureCrypto) } diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/store/memory/InMemoryMultiDomainEventLog.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/store/memory/InMemoryMultiDomainEventLog.scala index d0dc95ba8..2e4370e5b 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/store/memory/InMemoryMultiDomainEventLog.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/store/memory/InMemoryMultiDomainEventLog.scala @@ -14,7 +14,7 @@ import com.daml.platform.akkastreams.dispatcher.SubSource.RangeSource import com.digitalasset.canton.concurrent.DirectExecutionContext import com.digitalasset.canton.config.ProcessingTimeout import com.digitalasset.canton.data.CantonTimestamp -import com.digitalasset.canton.lifecycle.{FlagCloseable, Lifecycle} +import com.digitalasset.canton.lifecycle.{AsyncCloseable, FlagCloseable, Lifecycle} import com.digitalasset.canton.logging.{ErrorLoggingContext, NamedLoggerFactory, NamedLogging} import com.digitalasset.canton.participant.event.RecordOrderPublisher.{ PendingEventPublish, @@ -92,7 +92,7 @@ class InMemoryMultiDomainEventLog( ) ) - private val dispatcher = Dispatcher[GlobalOffset]( + private val dispatcher: Dispatcher[GlobalOffset] = Dispatcher[GlobalOffset]( loggerFactory.name, ledgerFirstOffset - 1, // start index is exclusive ledgerFirstOffset - 1, // end index is inclusive @@ -387,7 +387,11 @@ class InMemoryMultiDomainEventLog( Lifecycle.close( executionQueue .asCloseable("InMemoryMultiDomainEventLog.executionQueue", timeouts.shutdownShort.duration), - dispatcher, + AsyncCloseable( + s"${this.getClass}: dispatcher", + dispatcher.shutdown(), + timeouts.shutdownShort.duration, + ), )(logger) } diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/sync/SyncDomain.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/sync/SyncDomain.scala index f8ff64cd5..db03894c1 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/sync/SyncDomain.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/sync/SyncDomain.scala @@ -134,7 +134,7 @@ class SyncDomain( val staticDomainParameters: StaticDomainParameters = domainHandle.staticParameters private val seedGenerator = - new SeedGenerator(domainCrypto.crypto.privateCrypto, domainCrypto.crypto.pureCrypto) + new SeedGenerator(domainCrypto.crypto.pureCrypto) private val requestGenerator = ConfirmationRequestFactory(participantId, domainId)( @@ -220,13 +220,13 @@ class SyncDomain( pruneObserver.observer(_, _), killSwitch = selfKillSwitch, pruningMetrics, + staticDomainParameters.protocolVersion, timeouts, loggerFactory, ) ephemeral.recordOrderPublisher.setAcsChangeListener(listener) listener } - val topologyProcessor = new TopologyTransactionProcessor( domainId, domainCrypto.pureCrypto, @@ -257,6 +257,7 @@ class SyncDomain( domainCrypto, sequencerClient, participantId, + staticDomainParameters.protocolVersion, timeouts, loggerFactory, ) @@ -343,12 +344,12 @@ class SyncDomain( timestamp: CantonTimestamp ): EitherT[Future, SyncDomainInitializationError, Unit] = { val store = domainHandle.topologyStore - EitherT.right(store.findEffectiveTimestampsSince(timestamp).map { timestamps => - timestamps.headOption.foreach { head => + EitherT.right(store.findUpcomingEffectiveChanges(timestamp).map { changes => + changes.headOption.foreach { head => logger.debug( - s"Initialising the acs commitment processor with ${timestamps.length} effective times starting from: $head" + s"Initialising the acs commitment processor with ${changes.length} effective times starting from: ${head.effective}" ) - acsCommitmentProcessor.initializeTicksOnStartup(timestamps.toList) + acsCommitmentProcessor.initializeTicksOnStartup(changes.map(_.effective.value).toList) } }) } diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyDispatcher.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyDispatcher.scala index 16dde1cf6..774edec81 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyDispatcher.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyDispatcher.scala @@ -188,7 +188,7 @@ private class DomainOutbox( watermarkTsO <- targetStore.currentDispatchingWatermark watermarkTs = watermarkTsO.getOrElse(CantonTimestamp.MinValue) authorizedTsO <- authorizedStore.timestamp() - authorizedTs = authorizedTsO.getOrElse(CantonTimestamp.MinValue) + authorizedTs = authorizedTsO.map(_._2.value).getOrElse(CantonTimestamp.MinValue) // update cached watermark } yield { val cur = watermarks.updateAndGet { c => diff --git a/community/participant/src/main/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyManager.scala b/community/participant/src/main/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyManager.scala index eec6a9cf4..c72d0cc31 100644 --- a/community/participant/src/main/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyManager.scala +++ b/community/participant/src/main/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyManager.scala @@ -279,7 +279,7 @@ class ParticipantTopologyManager( filterNamespace = None, ) .map { current => - current.adds.toIdentityState + current.adds.toTopologyState .foldLeft(packages) { case (acc, TopologyStateUpdateElement(_, vs: VettedPackages)) => acc -- vs.packageIds diff --git a/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/MessageDispatcherTest.scala b/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/MessageDispatcherTest.scala index 66cb7df6a..6c9d66a7a 100644 --- a/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/MessageDispatcherTest.scala +++ b/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/MessageDispatcherTest.scala @@ -14,13 +14,7 @@ import com.digitalasset.canton.lifecycle.{FutureUnlessShutdown, UnlessShutdown} import com.digitalasset.canton.logging.{LogEntry, NamedLoggerFactory} import com.digitalasset.canton.participant.RequestCounter import com.digitalasset.canton.participant.event.RecordOrderPublisher -import com.digitalasset.canton.participant.protocol.MessageDispatcher.{ - DoNotExpectMediatorResult, - ExpectMalformedMediatorRequestResult, - MalformedMediatorRequestMessage, - SendMalformedAndExpectMediatorResult, - _, -} +import com.digitalasset.canton.participant.protocol.MessageDispatcher._ import com.digitalasset.canton.participant.protocol.conflictdetection.RequestTracker import com.digitalasset.canton.participant.protocol.submission.{ InFlightSubmissionTracker, @@ -355,6 +349,7 @@ trait MessageDispatcherTest { this: AsyncWordSpecLike with BaseTest => domainId, TestViewType, reject, + ProtocolVersion.latestForTest, ), dummySignature, ) @@ -1003,6 +998,7 @@ trait MessageDispatcherTest { this: AsyncWordSpecLike with BaseTest => domainId, viewType, reject, + ProtocolVersion.latestForTest, ), dummySignature, ) @@ -1105,8 +1101,7 @@ object MessageDispatcherTest { ) override def hashPurpose: HashPurpose = HashPurposeTest.testHashPurpose override def deserializedFrom: Option[ByteString] = None - override protected[this] def toByteStringUnmemoized(version: ProtocolVersion): ByteString = - ByteString.EMPTY + override protected[this] def toByteStringUnmemoized: ByteString = ByteString.EMPTY } } diff --git a/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/submission/ConfirmationRequestFactoryTest.scala b/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/submission/ConfirmationRequestFactoryTest.scala index 4b2b53dab..ecc08c84d 100644 --- a/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/submission/ConfirmationRequestFactoryTest.scala +++ b/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/submission/ConfirmationRequestFactoryTest.scala @@ -76,7 +76,6 @@ class ConfirmationRequestFactoryTest extends AsyncWordSpec with BaseTest with Ha .withReversedTopology(map) .withDomains(domain) .withKeyPurposes(keyPurposes) - .withHkdfOps(Some(new TestHkdf)) .build(loggerFactory) .forOwnerAndDomain(submitterParticipant, domain) .currentSnapshotApproximation @@ -100,12 +99,12 @@ class ConfirmationRequestFactoryTest extends AsyncWordSpec with BaseTest with Ha .build() .forOwnerAndDomain(submitterParticipant, domain) .currentSnapshotApproximation - val hashOps: HashOps = new SymbolicPureCrypto + val randomOps: RandomOps = new SymbolicPureCrypto val transactionUuid: UUID = new UUID(10L, 20L) val seedGenerator: SeedGenerator = - new SeedGenerator(privateCryptoApi.crypto.privateCrypto, hashOps) { + new SeedGenerator(randomOps) { override def generateUuid(): UUID = transactionUuid } @@ -121,7 +120,7 @@ class ConfirmationRequestFactoryTest extends AsyncWordSpec with BaseTest with Ha _confirmationPolicy: ConfirmationPolicy, _workflowId: Option[WorkflowId], _mediatorId: MediatorId, - transactionSeed: Salt, + transactionSeed: SaltSeed, transactionUuid: UUID, _topologySnapshot: TopologySnapshot, _contractOfId: SerializableContractOfId, @@ -140,15 +139,7 @@ class ConfirmationRequestFactoryTest extends AsyncWordSpec with BaseTest with Ha fail( s"Wrong transaction UUID $transactionUuid. Expected ${ConfirmationRequestFactoryTest.this.transactionUuid}" ) - EitherT(for { - expected <- seedGenerator - .generateSeedForTransaction(submitterInfo.changeId, domain, ledgerTime, transactionUuid) - .valueOrFail("generate expected seed") - } yield { - if (transactionSeed != expected) - fail(s"""Wrong transaction seed $transactionSeed. Expected $expected""".stripMargin) - transactionTreeFactoryResult - }) + transactionTreeFactoryResult.toEitherT } override def tryReconstruct( @@ -211,8 +202,8 @@ class ConfirmationRequestFactoryTest extends AsyncWordSpec with BaseTest with Ha } else None val keySeed = tree.viewPosition.position.foldRight(testKeySeed) { case (pos, seed) => - cryptoPureApi - .hkdfExpand( + ProtocolCryptoApi + .hkdf(cryptoPureApi, ProtocolVersion.latestForTest)( seed, cryptoPureApi.defaultSymmetricKeyScheme.keySizeInBytes, HkdfInfo.subview(pos), @@ -220,9 +211,12 @@ class ConfirmationRequestFactoryTest extends AsyncWordSpec with BaseTest with Ha .valueOr(e => throw new IllegalStateException(s"Failed to derive key: $e")) } - val keyLength = cryptoPureApi.defaultSymmetricKeyScheme.keySizeInBytes - val symmetricKey = cryptoPureApi - .hkdfExpand(keySeed, keyLength, HkdfInfo.ViewKey) + val symmetricKey = ProtocolCryptoApi + .hkdf(cryptoPureApi, ProtocolVersion.latestForTest)( + keySeed, + cryptoPureApi.defaultSymmetricKeyScheme.keySizeInBytes, + HkdfInfo.ViewKey, + ) .valueOr(e => fail(s"Failed to derive key: $e")) val participants = tree.informees @@ -266,7 +260,7 @@ class ConfirmationRequestFactoryTest extends AsyncWordSpec with BaseTest with Ha ) } - val testKeySeed = SecureRandomness.secureRandomness(0) + val testKeySeed = randomOps.generateSecureRandomness(0) def randomnessMap( randomness: SecureRandomness, diff --git a/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/submission/SeedGeneratorTest.scala b/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/submission/SeedGeneratorTest.scala index 7353ff8cd..723d48e87 100644 --- a/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/submission/SeedGeneratorTest.scala +++ b/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/submission/SeedGeneratorTest.scala @@ -3,42 +3,21 @@ package com.digitalasset.canton.participant.protocol.submission -import com.digitalasset.canton.data.CantonTimestamp -import com.digitalasset.canton.topology.{DefaultTestIdentities, DomainId, TestingIdentityFactory} -import com.digitalasset.canton.participant.protocol.submission.SeedGenerator.SeedForTransaction -import com.digitalasset.canton.{BaseTest, DefaultDamlValues} -import com.digitalasset.platform.daml.lf.testing.SampleParties.{Alice, Bob} +import com.digitalasset.canton.BaseTest +import com.digitalasset.canton.topology.{DefaultTestIdentities, TestingIdentityFactory} import org.scalatest.wordspec.AsyncWordSpec -import java.util.UUID - class SeedGeneratorTest extends AsyncWordSpec with BaseTest { - private val domain = DomainId(DefaultTestIdentities.uid) - "SeedGenerator" should { - "generate hashing salt based on command Id and other properties of the transaction" in { + "generate fresh salts" in { val crypto = TestingIdentityFactory(loggerFactory).newCrypto(DefaultTestIdentities.mediator) - val seedGenerator = new SeedGenerator(crypto.privateCrypto, crypto.pureCrypto) - val uuid = new UUID(0L, 1L) - val seedData = - SeedForTransaction( - DefaultDamlValues.changeId(Set(Alice, Bob)), - domain, - CantonTimestamp.Epoch, - uuid, - ).toDeterministicByteString - for { - seed <- seedGenerator - .generateSeedForTransaction( - DefaultDamlValues.changeId(Set(Alice, Bob)), - domain, - CantonTimestamp.Epoch, - uuid, - ) - .valueOrFail("generate seed") - hmac <- crypto.privateCrypto.hmac(seedData).valueOrFail("compute hmac") - } yield seed.unwrap shouldEqual hmac.unwrap + val seedGenerator = new SeedGenerator(crypto.pureCrypto) + + val salt1 = seedGenerator.generateSaltSeed() + val salt2 = seedGenerator.generateSaltSeed() + + salt1 should not equal salt2 } } } diff --git a/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/transfer/TransferInProcessingStepsTest.scala b/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/transfer/TransferInProcessingStepsTest.scala index c006f0a97..daa7c5cc0 100644 --- a/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/transfer/TransferInProcessingStepsTest.scala +++ b/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/transfer/TransferInProcessingStepsTest.scala @@ -115,7 +115,7 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { private val vault = crypto.privateCrypto val hash = TestHash.digest("123") - private val seedGenerator = new SeedGenerator(vault, pureCrypto) + private val seedGenerator = new SeedGenerator(pureCrypto) val globalTracker = new GlobalCausalOrderer( participant, _ => true, @@ -218,23 +218,21 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { TimeProofTestUtil.mkTimeProof(timestamp = CantonTimestamp.Epoch, domainId = targetDomain), ) val uuid = new UUID(1L, 2L) + val seed = seedGenerator.generateSaltSeed() + val transferData2 = { + val fullTransferOutTree = + transferOutRequest.toFullTransferOutTree(pureCrypto, pureCrypto, seed, uuid) + TransferData( + transferId.requestTimestamp, + 0L, + fullTransferOutTree, + CantonTimestamp.ofEpochSecond(10), + contract, + transactionId1, + None, + ) + } for { - seed <- seedGenerator - .generateSeedForTransferOut(transferOutRequest, uuid) - .valueOrFail("generate seed") - transferData2 = { - val fullTransferOutTree = - transferOutRequest.toFullTransferOutTree(pureCrypto, pureCrypto, seed, uuid) - TransferData( - transferId.requestTimestamp, - 0L, - fullTransferOutTree, - CantonTimestamp.ofEpochSecond(10), - contract, - transactionId1, - None, - ) - } deps <- statefulDependencies (persistentState, state) = deps _ <- setUpOrFail(transferData2, transferOutResult, persistentState) @@ -369,7 +367,7 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { pureCrypto, submitterParticipant, ) - val inTreeF = + val inTree = makeFullTransferInTree( party1, Set(party1), @@ -380,7 +378,6 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { transferOutResult, ) val inRequestF = for { - inTree <- inTreeF inRequest <- encryptFullTransferInTree(inTree) } yield inRequest @@ -400,7 +397,6 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { "succeed without errors" in { for { - inTree <- inTreeF inRequest <- inRequestF envelopes = NonEmpty(Seq, OpenEnvelope(inRequest, RecipientsTest.testInstance)) decrypted <- valueOrFail(transferInProcessingSteps.decryptViews(envelopes, cryptoSnapshot))( @@ -423,16 +419,16 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { } "fail when target domain is not current domain" in { + val inTree2 = makeFullTransferInTree( + party1, + Set(party1), + contract, + transactionId1, + anotherDomain, + anotherMediator, + transferOutResult, + ) for { - inTree2 <- makeFullTransferInTree( - party1, - Set(party1), - contract, - transactionId1, - anotherDomain, - anotherMediator, - transferOutResult, - ) result <- leftOrFail( transferInProcessingSteps.computeActivenessSetAndPendingContracts( CantonTimestamp.Epoch, @@ -456,7 +452,6 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { "fail when multiple requests are present" in { // Send the same transfer-in request twice for { - inTree <- inTreeF result <- leftOrFail( transferInProcessingSteps.computeActivenessSetAndPendingContracts( CantonTimestamp.Epoch, @@ -510,7 +505,7 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { (persistentState, ephemeralState) = deps // party2 is incorrectly registered as a stakeholder - fullTransferInTree2 <- makeFullTransferInTree( + fullTransferInTree2 = makeFullTransferInTree( party1, stakeholders = Set(party1, party2), contract, @@ -559,7 +554,7 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { transferLookup = ephemeralState.transferCache contractLookup = ephemeralState.contractLookup - fullTransferInTree <- makeFullTransferInTree( + fullTransferInTree = makeFullTransferInTree( party1, Set(party1), contract, @@ -616,7 +611,7 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { pureCrypto, submitterParticipant, ) - val inRequestF = + val inRequest = makeFullTransferInTree( party1, Set(party1), @@ -629,7 +624,6 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { "succeed without errors in the basic case" in { for { - inRequest <- inRequestF result <- valueOrFail( transferInProcessingSteps .validateTransferInRequest( @@ -657,13 +651,10 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { TimeProofTestUtil.mkTimeProof(timestamp = CantonTimestamp.Epoch, domainId = targetDomain), ) val uuid = new UUID(3L, 4L) - val transferDataF = for { - seed <- seedGenerator - .generateSeedForTransferOut(transferOutRequest, uuid) - .valueOrFail("generate seed") - } yield { - val fullTransferOutTree = - transferOutRequest.toFullTransferOutTree(pureCrypto, pureCrypto, seed, uuid) + val seed = seedGenerator.generateSaltSeed() + val fullTransferOutTree = + transferOutRequest.toFullTransferOutTree(pureCrypto, pureCrypto, seed, uuid) + val transferData = TransferData( CantonTimestamp.Epoch, 1L, @@ -673,12 +664,9 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { transactionId1, Some(transferOutResult), ) - } "succeed without errors when transfer data is valid" in { for { - inRequest <- inRequestF - transferData <- transferDataF result <- valueOrFail( transferInProcessingSteps .validateTransferInRequest( @@ -709,26 +697,23 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { Some(promise.future), // Topology state is not available ) - for { - inRequest <- inRequestF - transferData <- transferDataF - inValidated = transferInProcessingSteps2 - .validateTransferInRequest( - CantonTimestamp.Epoch, - inRequest, - Some(transferData), - cryptoSnapshot, - transferringParticipant = false, - ) - .value + val inValidated = transferInProcessingSteps2 + .validateTransferInRequest( + CantonTimestamp.Epoch, + inRequest, + Some(transferData), + cryptoSnapshot, + transferringParticipant = false, + ) + .value - _ = always() { - inValidated.isCompleted shouldBe false - } + always() { + inValidated.isCompleted shouldBe false + } - _ = promise.completeWith(Future.unit) + promise.completeWith(Future.unit) + for { completed <- inValidated - } yield { succeed } } } @@ -810,7 +795,7 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { EitherT.pure[Future, Error](ContractMetadata.tryCreate(signatories, stakeholders, None)) } } - val seedGenerator = new SeedGenerator(vault, pureCrypto) + val seedGenerator = new SeedGenerator(pureCrypto) new TransferInProcessingSteps( domainId, @@ -839,12 +824,9 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { targetMediator: MediatorId, transferOutResult: DeliveredTransferOutResult, uuid: UUID = new UUID(4L, 5L), - ): Future[FullTransferInTree] = - for { - seed <- seedGenerator - .generateSeedForTransferIn(contract.contractId, transferOutResult, targetDomain, uuid) - .valueOrFail("generate seed") - } yield TransferInProcessingSteps.makeFullTransferInTree( + ): FullTransferInTree = { + val seed = seedGenerator.generateSaltSeed() + TransferInProcessingSteps.makeFullTransferInTree( pureCrypto, seed, submitter, @@ -856,6 +838,7 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { transferOutResult, uuid, ) + } def encryptFullTransferInTree( tree: FullTransferInTree @@ -875,19 +858,21 @@ class TransferInProcessingStepsTest extends AsyncWordSpec with BaseTest { targetDomain: DomainId, targetMediator: MediatorId, transferOutResult: DeliveredTransferOutResult, - ): Future[EncryptedViewMessage[TransferInViewType]] = + ): Future[EncryptedViewMessage[TransferInViewType]] = { + val tree = makeFullTransferInTree( + submitter, + stakeholders, + contract, + creatingTransactionId, + targetDomain, + targetMediator, + transferOutResult, + ) + for { - tree <- makeFullTransferInTree( - submitter, - stakeholders, - contract, - creatingTransactionId, - targetDomain, - targetMediator, - transferOutResult, - ) request <- encryptFullTransferInTree(tree) } yield request + } def makeRootHashMessage( tree: FullTransferInTree @@ -923,6 +908,7 @@ object TransferInProcessingStepsTest { Set(), TransferOutDomainId(originDomain), Verdict.Approve, + ProtocolVersion.latestForTest, ) val signedResult: SignedProtocolMessage[TransferOutResult] = Await.result(SignedProtocolMessage.tryCreate(result, cryptoSnapshot, hashOps), 10.seconds) @@ -950,12 +936,11 @@ object TransferInProcessingStepsTest { transferOutResult } - def transferInResult(targetDomain: DomainId): TransferInResult = { - TransferResult.create( - RequestId(CantonTimestamp.Epoch), - Set(), - TransferInDomainId(targetDomain), - Verdict.Approve, - ) - } + def transferInResult(targetDomain: DomainId): TransferInResult = TransferResult.create( + RequestId(CantonTimestamp.Epoch), + Set(), + TransferInDomainId(targetDomain), + Verdict.Approve, + ProtocolVersion.latestForTest, + ) } diff --git a/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutProcessingStepsTest.scala b/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutProcessingStepsTest.scala index 4b3e29ae7..2018a3198 100644 --- a/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutProcessingStepsTest.scala +++ b/community/participant/src/test/scala/com/digitalasset/canton/participant/protocol/transfer/TransferOutProcessingStepsTest.scala @@ -174,7 +174,7 @@ class TransferOutProcessingStepsTest extends AsyncWordSpec with BaseTest with Ha .forOwnerAndDomain(submittingParticipant, originDomain) .currentSnapshotApproximation - val seedGenerator = new SeedGenerator(cryptoSnapshot.crypto.privateCrypto, pureCrypto) + val seedGenerator = new SeedGenerator(pureCrypto) private val coordination: TransferCoordination = TestTransferCoordination( @@ -484,9 +484,8 @@ class TransferOutProcessingStepsTest extends AsyncWordSpec with BaseTest with Ha targetDomain, timeEvent, ) - val outTreeF = makeFullTransferOutTree(outRequest) + val outTree = makeFullTransferOutTree(outRequest) val encryptedOutRequestF = for { - outTree <- outTreeF encrypted <- encryptTransferOutTree(outTree) } yield encrypted @@ -506,7 +505,6 @@ class TransferOutProcessingStepsTest extends AsyncWordSpec with BaseTest with Ha "succeed without errors" in { for { - outTree <- outTreeF encryptedOutRequest <- encryptedOutRequestF envelopes = NonEmpty(Seq, OpenEnvelope(encryptedOutRequest, RecipientsTest.testInstance)) decrypted <- valueOrFail(outProcessingSteps.decryptViews(envelopes, cryptoSnapshot))( @@ -561,16 +559,16 @@ class TransferOutProcessingStepsTest extends AsyncWordSpec with BaseTest with Ha timeEvent, ) + val fullTransferOutTree = makeFullTransferOutTree(outRequest) + val dataAndResponseArgs = TransferOutProcessingSteps.PendingDataAndResponseArgs( + fullTransferOutTree, + Recipients.cc(submittingParticipant), + CantonTimestamp.Epoch, + 1L, + 1L, + cryptoSnapshot, + ) for { - fullTransferOutTree <- makeFullTransferOutTree(outRequest) - dataAndResponseArgs = TransferOutProcessingSteps.PendingDataAndResponseArgs( - fullTransferOutTree, - Recipients.cc(submittingParticipant), - CantonTimestamp.Epoch, - 1L, - 1L, - cryptoSnapshot, - ) _ <- state.storedContractManager.addPendingContracts( 1L, Seq(WithTransactionId(contract, transactionId)), @@ -603,6 +601,7 @@ class TransferOutProcessingStepsTest extends AsyncWordSpec with BaseTest with Ha Set(), TransferOutDomainId(originDomain), Verdict.Approve, + ProtocolVersion.latestForTest, ) for { signedResult <- SignedProtocolMessage.tryCreate(transferResult, cryptoSnapshot, pureCrypto) @@ -655,10 +654,9 @@ class TransferOutProcessingStepsTest extends AsyncWordSpec with BaseTest with Ha def makeFullTransferOutTree( request: TransferOutRequest, uuid: UUID = new UUID(6L, 7L), - ): Future[FullTransferOutTree] = { - for { - seed <- seedGenerator.generateSeedForTransferOut(request, uuid).valueOrFail("generate seed") - } yield request.toFullTransferOutTree(pureCrypto, pureCrypto, seed, uuid) + ): FullTransferOutTree = { + val seed = seedGenerator.generateSaltSeed() + request.toFullTransferOutTree(pureCrypto, pureCrypto, seed, uuid) } def encryptTransferOutTree( diff --git a/community/participant/src/test/scala/com/digitalasset/canton/participant/pruning/AcsCommitmentProcessorTest.scala b/community/participant/src/test/scala/com/digitalasset/canton/participant/pruning/AcsCommitmentProcessorTest.scala index 520440162..831953258 100644 --- a/community/participant/src/test/scala/com/digitalasset/canton/participant/pruning/AcsCommitmentProcessorTest.scala +++ b/community/participant/src/test/scala/com/digitalasset/canton/participant/pruning/AcsCommitmentProcessorTest.scala @@ -95,6 +95,8 @@ trait AcsCommitmentProcessorBaseTest extends BaseTest { remoteId2 -> Set(carol), ) + protected val protocolVersion = TestDomainParameters.defaultStatic.protocolVersion + def ts(i: Int): CantonTimestampSecond = CantonTimestampSecond.ofEpochSecond(i.longValue) def toc(timestamp: Int, requestCounter: Int = 0): TimeOfChange = @@ -176,9 +178,10 @@ trait AcsCommitmentProcessorBaseTest extends BaseTest { .thenReturn(EitherT.rightT[Future, SendAsyncClientError](())) val changeTimes = - (timeProofs.map(ts => TimeOfChange(0, ts)) ++ contractSetup.values.toList.flatMap(cInfo => - List(cInfo._2, cInfo._3) - )).distinct.sorted + (timeProofs.map(ts => TimeOfChange(0, ts)) ++ contractSetup.values.toList.flatMap { + case (_, creationTs, archivalTs) => + List(creationTs, archivalTs) + }).distinct.sorted val changes = changeTimes.map(changesAtToc(contractSetup)) val store = optCommitmentStore.getOrElse(new InMemoryAcsCommitmentStore(loggerFactory)) val acsCommitmentProcessor = new AcsCommitmentProcessor( @@ -191,6 +194,7 @@ trait AcsCommitmentProcessorBaseTest extends BaseTest { (_, _) => FutureUnlessShutdown.unit, killSwitch, ParticipantTestMetrics.pruning, + protocolVersion, DefaultProcessingTimeouts.testing .copy(storageMaxRetryInterval = TimeoutDuration.tryFromDuration(1.millisecond)), loggerFactory, @@ -514,7 +518,14 @@ class AcsCommitmentProcessorTest extends AsyncWordSpec with AcsCommitmentProcess val snapshotF = crypto.snapshot(CantonTimestamp.Epoch) val period = CommitmentPeriod(fromExclusive.forgetSecond, toInclusive.forgetSecond, interval).value - val payload = AcsCommitment.create(domainId, remote, localId, period, cmt) + val payload = AcsCommitment.create( + domainId, + remote, + localId, + period, + cmt, + protocolVersion, + ) snapshotF.flatMap { snapshot => SignedProtocolMessage.tryCreate(payload, snapshot, crypto.pureCrypto) diff --git a/community/participant/src/test/scala/com/digitalasset/canton/participant/store/AcsCommitmentStoreTest.scala b/community/participant/src/test/scala/com/digitalasset/canton/participant/store/AcsCommitmentStoreTest.scala index 86af4e7e7..d44307178 100644 --- a/community/participant/src/test/scala/com/digitalasset/canton/participant/store/AcsCommitmentStoreTest.scala +++ b/community/participant/src/test/scala/com/digitalasset/canton/participant/store/AcsCommitmentStoreTest.scala @@ -13,10 +13,11 @@ import com.digitalasset.canton.protocol.messages.{ CommitmentPeriod, SignedProtocolMessage, } -import com.digitalasset.canton.protocol.ContractMetadata +import com.digitalasset.canton.protocol.{ContractMetadata, TestDomainParameters} import com.digitalasset.canton.store.PrunableByTimeTest import com.digitalasset.canton.time.PositiveSeconds import com.digitalasset.canton.util.FutureUtil +import com.digitalasset.canton.version.ProtocolVersion import com.digitalasset.canton.{BaseTest, LfPartyId} import com.google.protobuf.ByteString import org.scalatest.wordspec.AsyncWordSpec @@ -31,9 +32,11 @@ trait CommitmentStoreBaseTest extends AsyncWordSpec with BaseTest { val symbolicVault = SymbolicCrypto - .tryCreate(Seq(Fingerprint.tryCreate("test")), Seq(), None, timeouts, loggerFactory) + .tryCreate(Seq(Fingerprint.tryCreate("test")), Seq(), timeouts, loggerFactory) .privateCrypto + protected val protocolVersion = TestDomainParameters.defaultStatic.protocolVersion + val localId = ParticipantId(UniqueIdentifier.tryFromProtoPrimitive("localParticipant::domain")) val remoteId = ParticipantId(UniqueIdentifier.tryFromProtoPrimitive("remoteParticipant::domain")) val remoteId2 = ParticipantId( @@ -72,7 +75,14 @@ trait CommitmentStoreBaseTest extends AsyncWordSpec with BaseTest { .valueOrFail("failed to create dummy signature") val dummyCommitmentMsg = - AcsCommitment.create(domainId, remoteId, localId, period(0, 1), dummyCommitment) + AcsCommitment.create( + domainId, + remoteId, + localId, + period(0, 1), + dummyCommitment, + protocolVersion, + ) val dummySigned = SignedProtocolMessage(dummyCommitmentMsg, dummySignature) val alice: LfPartyId = LfPartyId.assertFromString("Alice") @@ -275,11 +285,23 @@ trait AcsCommitmentStoreTest extends CommitmentStoreBaseTest with PrunableByTime "correctly search stored remote commitment messages" in { val store = mk() - val dummyMsg2 = - AcsCommitment.create(domainId, remoteId, localId, period(2, 3), dummyCommitment) + val dummyMsg2 = AcsCommitment.create( + domainId, + remoteId, + localId, + period(2, 3), + dummyCommitment, + protocolVersion, + ) val dummySigned2 = SignedProtocolMessage(dummyMsg2, dummySignature) - val dummyMsg3 = - AcsCommitment.create(domainId, remoteId2, localId, period(0, 1), dummyCommitment) + val dummyMsg3 = AcsCommitment.create( + domainId, + remoteId2, + localId, + period(0, 1), + dummyCommitment, + protocolVersion, + ) val dummySigned3 = SignedProtocolMessage(dummyMsg3, dummySignature) for { @@ -297,8 +319,14 @@ trait AcsCommitmentStoreTest extends CommitmentStoreBaseTest with PrunableByTime "allow storing different remote commitment messages for the same period" in { val store = mk() - val dummyMsg2 = - AcsCommitment.create(domainId, remoteId, localId, period(0, 1), dummyCommitment2) + val dummyMsg2 = AcsCommitment.create( + domainId, + remoteId, + localId, + period(0, 1), + dummyCommitment2, + protocolVersion, + ) val dummySigned2 = SignedProtocolMessage(dummyMsg2, dummySignature) for { @@ -498,7 +526,7 @@ trait CommitmentQueueTest extends CommitmentStoreBaseTest { localId, CommitmentPeriod(ts(start), ts(end), PositiveSeconds.ofSeconds(5)).value, cmt, - )(None) + )(ProtocolVersion.latestForTest, None) "work sensibly in a basic scenario" in { val queue = mk() diff --git a/community/participant/src/test/scala/com/digitalasset/canton/participant/store/TransferStoreTest.scala b/community/participant/src/test/scala/com/digitalasset/canton/participant/store/TransferStoreTest.scala index 4c78abee8..c21040770 100644 --- a/community/participant/src/test/scala/com/digitalasset/canton/participant/store/TransferStoreTest.scala +++ b/community/participant/src/test/scala/com/digitalasset/canton/participant/store/TransferStoreTest.scala @@ -23,6 +23,7 @@ import com.digitalasset.canton.sequencing.protocol._ import com.digitalasset.canton.time.TimeProofTestUtil import com.digitalasset.canton.topology._ import com.digitalasset.canton.util.{Checked, FutureUtil} +import com.digitalasset.canton.version.ProtocolVersion import com.digitalasset.canton.{BaseTest, LfPartyId} import org.scalatest.wordspec.AsyncWordSpec @@ -602,7 +603,7 @@ object TransferStoreTest { 10.seconds, ) } - val seedGenerator = new SeedGenerator(privateCrypto, pureCryptoApi) + val seedGenerator = new SeedGenerator(pureCryptoApi) def mkTransferDataForDomain( transferId: TransferId, @@ -621,13 +622,10 @@ object TransferStoreTest { TimeProofTestUtil.mkTimeProof(timestamp = CantonTimestamp.Epoch, domainId = targetDomainId), ) val uuid = new UUID(10L, 0L) - for { - seed <- seedGenerator - .generateSeedForTransferOut(transferOutRequest, uuid) - .valueOr(err => throw new IllegalStateException(err.toString)) - } yield { - val fullTransferOutViewTree = - transferOutRequest.toFullTransferOutTree(pureCryptoApi, pureCryptoApi, seed, uuid) + val seed = seedGenerator.generateSaltSeed() + val fullTransferOutViewTree = + transferOutRequest.toFullTransferOutTree(pureCryptoApi, pureCryptoApi, seed, uuid) + Future.successful( TransferData( transferId.requestTimestamp, 0L, @@ -637,7 +635,7 @@ object TransferStoreTest { transactionId1, None, ) - } + ) } private def mkTransferData( @@ -656,6 +654,7 @@ object TransferStoreTest { requestId, Verdict.Approve, mediatorMessage.allInformees, + ProtocolVersion.latestForTest, ) val signedResult = SignedProtocolMessage(result, sign("TransferOutResult-mediator")) val batch = Batch.of(signedResult -> RecipientsTest.testInstance) diff --git a/community/participant/src/test/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyDispatcherTest.scala b/community/participant/src/test/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyDispatcherTest.scala index 5f4033dbd..79c7120e5 100644 --- a/community/participant/src/test/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyDispatcherTest.scala +++ b/community/participant/src/test/scala/com/digitalasset/canton/participant/topology/ParticipantTopologyDispatcherTest.scala @@ -218,18 +218,18 @@ class ParticipantTopologyDispatcherTest extends AsyncWordSpec with BaseTest { // add a remove and another add _ <- push(manager, Seq(midRevert, another)) // ensure that topology manager properly processed this state - ais <- source.headTransactions.map(_.toIdentityState) + ais <- source.headTransactions.map(_.toTopologyState) _ = ais should not contain (midRevert.element) _ = ais should contain(another.element) // and ensure both are not in the new store - tis <- target.headTransactions.map(_.toIdentityState) + tis <- target.headTransactions.map(_.toTopologyState) _ = tis should contain(midRevert.element) _ = tis should not contain (another.element) // re-connect _ = handle.clear(2) _ <- dispatcherConnected(dispatcher, handle, client, target) _ <- handle.allObserved() - tis <- target.headTransactions.map(_.toIdentityState) + tis <- target.headTransactions.map(_.toTopologyState) } yield { tis should not contain (midRevert.element) tis should contain(another.element) diff --git a/project/BuildCommon.scala b/project/BuildCommon.scala index 0993c4dba..950362096 100644 --- a/project/BuildCommon.scala +++ b/project/BuildCommon.scala @@ -614,6 +614,8 @@ object BuildCommon { daml_lf_archive_reader, daml_lf_dev_archive_java_proto, daml_lf_engine, + daml_ledger_api_auth_client, + daml_participant_integration_api, logback_classic % Runtime, logback_core % Runtime, akka_stream, @@ -706,23 +708,7 @@ object BuildCommon { .disablePlugins(WartRemover) // to accommodate different daml repo coding style .settings( sharedSettings, - libraryDependencies ++= Seq( - log4j_core, - log4j_api, - logback_classic, - logback_core, - daml_participant_integration_api excludeAll (ExclusionRule( - "org.apache.logging.log4j" // as a way to get all third party dependencies conveniently plus code not copied - ), - ExclusionRule( - "ch.qos.logback" // do not bring in older transient dependency) - )), - daml_contextualized_logging, - daml_error, - daml_metrics, - daml_ledger_api_scalapb, - scalapb_runtime, - ), + libraryDependencies ++= Seq(), dependencyOverrides ++= Seq(log4j_core, log4j_api), Compile / PB.targets := Seq( scalapb.gen(flatPackage = diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b54a1df40..c2750bc6e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -78,12 +78,12 @@ object Dependencies { lazy val daml_nonempty_cats = "com.daml" %% "nonempty-cats" % daml_libraries_version lazy val daml_ledger_api_common = "com.daml" %% "ledger-api-common" % daml_libraries_version + lazy val daml_ledger_api_auth_client = + "com.daml" % "ledger-api-auth-client" % daml_libraries_version lazy val daml_ledger_api_client = "com.daml" %% "ledger-api-client" % daml_libraries_version lazy val daml_participant_integration_api = "com.daml" %% "participant-integration-api" % daml_libraries_version lazy val daml_caching = "com.daml" %% "caching" % daml_libraries_version - lazy val daml_contextualized_logging = - "com.daml" %% "contextualized-logging" % daml_libraries_version lazy val daml_error = "com.daml" %% "error" % daml_libraries_version lazy val daml_error_generator = "com.daml" %% "error-generator-lib" % daml_libraries_version lazy val daml_metrics = "com.daml" %% "metrics" % daml_libraries_version @@ -91,7 +91,6 @@ object Dependencies { lazy val daml_resources_akka = "com.daml" %% "resources-akka" % daml_libraries_version lazy val daml_ledger_rxjava_client = "com.daml" % "bindings-rxjava" % daml_libraries_version - lazy val daml_ledger_api_scalapb = "com.daml" %% "ledger-api-scalapb" % daml_libraries_version lazy val da_grpc_bindings_ledger_client = "com.daml" %% "bindings-scala" % daml_libraries_version lazy val da_akka_bindings = "com.daml" %% "bindings-akka" % daml_libraries_version diff --git a/project/project/DamlVersions.scala b/project/project/DamlVersions.scala index 6d0cb47af..b2dd6823b 100644 --- a/project/project/DamlVersions.scala +++ b/project/project/DamlVersions.scala @@ -7,15 +7,15 @@ object DamlVersions { /** The version of the daml compiler (and in most cases of the daml libraries as well). */ - val version: String = "2.2.0-snapshot.20220504.9851.0.4c8e027d" + val version: String = "2.3.0-snapshot.20220511.9887.0.5a509164" /** Custom Daml artifacts override version. */ - private val customDamlVersion: String = "0.0.0" + private val customDamlVersion: String = sys.env.getOrElse("DAML_HEAD_VERSION", "0.0.0") /** Whether we want to use the customDamlVersion. */ - val useCustomDamlVersion: Boolean = sys.env.contains("DAML_SNAPSHOT") + val useCustomDamlVersion: Boolean = sys.env.contains("DAML_HEAD_VERSION") /** The version to use when sdk jvm libraries published to maven repositories. */