[main] Update 2024-09-03.23 (#220)

Reference commit: 7341341db1 Co-authored-by: Canton <[email protected]>
digital-asset · Sep 4, 2024 · 6fa7435 · 6fa7435
1 parent eb23bc2
commit 6fa7435
Show file tree

Hide file tree

Showing 469 changed files with 11,586 additions and 18,728 deletions.
diff --git a/UNRELEASED.md b/UNRELEASED.md
@@ -3,12 +3,37 @@
 Canton CANTON_VERSION has been released on RELEASE_DATE. You can download the Daml Open Source edition from the Daml Connect [Github Release Section](https://github.com/digital-asset/daml/releases/tag/vCANTON_VERSION). The Enterprise edition is available on [Artifactory](https://digitalasset.jfrog.io/artifactory/canton-enterprise/canton-enterprise-CANTON_VERSION.zip).
 Please also consult the [full documentation of this release](https://docs.daml.com/CANTON_VERSION/canton/about.html).
 
+## Until 2024-09-04 (Exclusive)
+
+- google.protobuf.XValue wrapper messages are replaced by `optional X` in the protobuf definitions. Incompatibility for manually crafted Protobuf messages and wire formats. Protobuf bindings must be regenerated, but should remain compatible.
+- Started the renaming transfer -> reassignment
+  - transferExclusivityTimeout -> assignmentExclusivityTimeout
+- Added periodic generation of sequencer counter checkpoints to the sequencer and reworked SQL queries.
+  - This should improve performance for sequencer snapshotting and pruning and reduce database load.
+  - The checkpoint interval is configurable under `canton.sequencers.<sequencer>.writer.checkpoint-interval` (default: 30s):
+```hocon
+writer {
+  checkpoint-interval = "30s"
+}
+```
+
+## Until 2024-08-30 (Exclusive)
+- The `ParticipantOffset` message was removed since it was already replaced by a simpler string representation and
+  was not used anymore.
+
 ## Until 2024-08-28 (Exclusive)
 - the DomainId field has been removed from the following topology mapping: `OwnerToKeyMapping`, `VettedPackages`, `PartyToParticipant` and `AuthorityOf`.
   Those fields were not handled properly, so we decide to remove them.
-- two new endpoints added to `GrpcInspectionService` to inspect the state of sent and received ACS commitments on participants. 
+- two new endpoints added to `GrpcInspectionService` to inspect the state of sent and received ACS commitments on participants.
   - `lookupSentAcsCommitments` to retrieve sent ACS Commitments and their states
   - `lookupReceivedAcsCommitments` to retrieve received ACS commitments and their states
+- When not specifying `AuthorizeRequest.signed_by` or `SignTransactionsRequest.signed_by`, suitable signing keys available to the node are selected automatically.
+
+## Until 2024-08-26 (Exclusive)
+
+### Changes in `VersionService.GetLedgerApiVersion`
+- The `GetLedgerApiVersion` method of the `VersionService` contains new `features.offset_checkpoint` field within the returned `GetLedgerApiVersionResponse` message.
+  It exposes the `max_offset_checkpoint_emission_delay` which is the maximum time needed to emit a new OffsetCheckpoint.
 
 ## Until 2024-08-21 (Exclusive)
 - Error INVALID_SUBMITTER is changed to INVALID_READER
@@ -33,6 +58,9 @@ We have introduced additional package vetting validations that may result in pac
 ### Mediators may not be in two mediator groups at the same time
 Add mediators to multiple groups results in a rejection with error `MEDIATORS_ALREADY_IN_OTHER_GROUPS`.
 
+### Traffic purchase handler returns early
+SetTrafficPurchased requests return immediately and no longer return the max sequencing time.
+
 ## Until 2024-07-31 (Exclusive)
 
 - Removed the GrpcTransferService

diff --git a/...api/src/main/protobuf/com/digitalasset/canton/admin/domain/v30/sequencer_connection.proto b/...api/src/main/protobuf/com/digitalasset/canton/admin/domain/v30/sequencer_connection.proto
@@ -6,7 +6,6 @@ syntax = "proto3";
 package com.digitalasset.canton.admin.domain.v30;
 
 import "google/protobuf/duration.proto";
-import "google/protobuf/wrappers.proto";
 import "scalapb/scalapb.proto";
 
 message SequencerConnection {
@@ -21,7 +20,7 @@ message SequencerConnection {
     repeated string connections = 1;
     // Should the connection use TLS?
     bool transportSecurity = 2;
-    google.protobuf.BytesValue customTrustCertificates = 3;
+    optional bytes customTrustCertificates = 3;
   }
 }
 

diff --git a/...admin-api/src/main/protobuf/com/digitalasset/canton/admin/health/v30/status_service.proto b/...admin-api/src/main/protobuf/com/digitalasset/canton/admin/health/v30/status_service.proto
@@ -6,7 +6,6 @@ syntax = "proto3";
 package com.digitalasset.canton.admin.health.v30;
 
 import "google/protobuf/duration.proto";
-import "google/protobuf/wrappers.proto";
 
 service StatusService {
   // Streams back a zip file as byte chunks, containing a health dump of the full canton process (including other nodes)
@@ -38,7 +37,7 @@ message ComponentStatus {
     StatusData fatal = 5;
   }
   message StatusData {
-    google.protobuf.StringValue description = 1;
+    optional string description = 1;
   }
 }
 
@@ -67,7 +66,7 @@ message Status {
 message HealthDumpRequest {
   // Size of the byte chunks to stream back.
   // Defaults to 2MB (half of the default gRPC max message size)
-  google.protobuf.UInt32Value chunk_size = 1;
+  optional uint32 chunk_size = 1;
 }
 
 message HealthDumpResponse {

diff --git a/.../src/main/protobuf/com/digitalasset/canton/admin/participant/v30/inspection_service.proto b/.../src/main/protobuf/com/digitalasset/canton/admin/participant/v30/inspection_service.proto
@@ -23,8 +23,6 @@ service InspectionService {
   // Look up the ledger offset corresponding to the timestamp, specifically the largest offset such that no later
   // offset corresponds to a later timestamp than the specified one.
   rpc LookupOffsetByTime(LookupOffsetByTime.Request) returns (LookupOffsetByTime.Response);
-  // Look up the ledger offset by an index, e.g. 1 returns the first offset, 2 the second, etc.
-  rpc LookupOffsetByIndex(LookupOffsetByIndex.Request) returns (LookupOffsetByIndex.Response);
   // Request metadata about shared contracts used in commitment computation at a specific time
   // Subject to the data still being available on the participant
   // TODO(#9557) R2
@@ -88,19 +86,6 @@ message LookupOffsetByTime {
   }
 }
 
-message LookupOffsetByIndex {
-  message Request {
-    // the index to look up the offset for, needs to be 1 or larger
-    int64 index = 1;
-  }
-
-  message Response {
-    // the absolute offset as a string corresponding to the specified index.
-    // empty string if no such offset exists.
-    string offset = 1;
-  }
-}
-
 message OpenCommitment {
   message Request {
     // the commitment to be opened

diff --git a/...n/admin/participant/v30/transfer_id.proto → ...min/participant/v30/reassignment_id.proto b/...n/admin/participant/v30/transfer_id.proto → ...min/participant/v30/reassignment_id.proto
@@ -7,7 +7,7 @@ package com.digitalasset.canton.admin.participant.v30;
 
 import "google/protobuf/timestamp.proto";
 
-message TransferId {
+message ReassignmentId {
   string source_domain = 1;
   google.protobuf.Timestamp timestamp = 2;
 }
diff --git a/...main/protobuf/com/digitalasset/canton/admin/participant/v30/traffic_control_service.proto b/...main/protobuf/com/digitalasset/canton/admin/participant/v30/traffic_control_service.proto
@@ -5,8 +5,6 @@ syntax = "proto3";
 
 package com.digitalasset.canton.admin.participant.v30;
 
-import "google/protobuf/wrappers.proto";
-
 /*
  * Service to retrieve information about the traffic state of the participant.
  */
@@ -36,5 +34,5 @@ message TrafficState {
   // Timestamp at which the state is valid
   int64 timestamp = 5;
   // Optional serial of the balance update that updated the extra traffic limit
-  google.protobuf.UInt32Value serial = 6;
+  optional uint32 serial = 6;
 }
diff --git a/.../src/main/scala/com/digitalasset/canton/admin/api/client/commands/LedgerApiCommands.scala b/.../src/main/scala/com/digitalasset/canton/admin/api/client/commands/LedgerApiCommands.scala
@@ -81,7 +81,6 @@ import com.daml.ledger.api.v2.event_query_service.{
   GetEventsByContractIdRequest,
   GetEventsByContractIdResponse,
 }
-import com.daml.ledger.api.v2.participant_offset.ParticipantOffset
 import com.daml.ledger.api.v2.reassignment.{AssignedEvent, Reassignment, UnassignedEvent}
 import com.daml.ledger.api.v2.reassignment_command.{
   AssignCommand,
@@ -416,22 +415,20 @@ object LedgerApiCommands {
 
     }
 
-    final case class Prune(pruneUpTo: ParticipantOffset)
+    final case class Prune(pruneUpTo: String)
         extends BaseCommand[PruneRequest, PruneResponse, Unit] {
 
       override def timeoutType: TimeoutType =
         DefaultUnboundedTimeout // pruning can take a very long time
 
       override def createRequest(): Either[String, PruneRequest] =
-        pruneUpTo.value.absolute
-          .toRight("The pruneUpTo ledger offset needs to be absolute")
-          .map(
-            PruneRequest(
-              _,
-              // canton always prunes divulged contracts both in the ledger api index-db and in canton stores
-              pruneAllDivulgedContracts = true,
-            )
+        Right(
+          PruneRequest(
+            pruneUpTo,
+            // canton always prunes divulged contracts both in the ledger api index-db and in canton stores
+            pruneAllDivulgedContracts = true,
           )
+        )
 
       override def submitRequest(
           service: ParticipantPruningServiceStub,
@@ -997,9 +994,7 @@ object LedgerApiCommands {
     sealed trait ReassignmentWrapper extends UpdateTreeWrapper with UpdateWrapper {
       def reassignment: Reassignment
       def unassignId: String = reassignment.getUnassignedEvent.unassignId
-      def offset: ParticipantOffset = ParticipantOffset(
-        ParticipantOffset.Value.Absolute(reassignment.offset)
-      )
+      def offset: String = reassignment.offset
     }
     object ReassignmentWrapper {
       def apply(reassignment: Reassignment): ReassignmentWrapper = {

diff --git a/...in/scala/com/digitalasset/canton/admin/api/client/commands/ParticipantAdminCommands.scala b/...in/scala/com/digitalasset/canton/admin/api/client/commands/ParticipantAdminCommands.scala
@@ -4,8 +4,6 @@
 package com.digitalasset.canton.admin.api.client.commands
 
 import cats.implicits.*
-import com.daml.ledger.api.v2.participant_offset.ParticipantOffset
-import com.daml.ledger.api.v2.participant_offset.ParticipantOffset.Value
 import com.digitalasset.canton.admin.api.client.commands.GrpcAdminCommand.{
   DefaultUnboundedTimeout,
   ServerEnforcedTimeout,
@@ -45,7 +43,6 @@ import com.digitalasset.canton.participant.pruning.AcsCommitmentProcessor.{
   SentCmtState,
   SharedContractsState,
 }
-import com.digitalasset.canton.participant.sync.UpstreamOffsetConvert
 import com.digitalasset.canton.protocol.LfContractId
 import com.digitalasset.canton.protocol.messages.{AcsCommitment, CommitmentPeriod}
 import com.digitalasset.canton.sequencing.SequencerConnectionValidation
@@ -1022,22 +1019,6 @@ object ParticipantAdminCommands {
         Right(response.offset)
     }
 
-    final case class LookupOffsetByIndex(index: Long)
-        extends Base[v30.LookupOffsetByIndex.Request, v30.LookupOffsetByIndex.Response, String] {
-      override def createRequest() = Right(v30.LookupOffsetByIndex.Request(index))
-
-      override def submitRequest(
-          service: InspectionServiceStub,
-          request: v30.LookupOffsetByIndex.Request,
-      ): Future[v30.LookupOffsetByIndex.Response] =
-        service.lookupOffsetByIndex(request)
-
-      override def handleResponse(
-          response: v30.LookupOffsetByIndex.Response
-      ): Either[String, String] =
-        Right(response.offset)
-    }
-
     // TODO(#9557) R2 The code below should be sufficient
     final case class OpenCommitment(
         observer: StreamObserver[v30.OpenCommitment.Response],
@@ -1496,7 +1477,7 @@ object ParticipantAdminCommands {
 
     final case class GetSafePruningOffsetCommand(beforeOrAt: Instant, ledgerEnd: String)
         extends Base[v30.GetSafePruningOffsetRequest, v30.GetSafePruningOffsetResponse, Option[
-          ParticipantOffset
+          String
         ]] {
 
       override def createRequest(): Either[String, v30.GetSafePruningOffsetRequest] =
@@ -1511,22 +1492,20 @@ object ParticipantAdminCommands {
 
       override def handleResponse(
           response: v30.GetSafePruningOffsetResponse
-      ): Either[String, Option[ParticipantOffset]] = response.response match {
+      ): Either[String, Option[String]] = response.response match {
         case v30.GetSafePruningOffsetResponse.Response.Empty => Left("Unexpected empty response")
 
         case v30.GetSafePruningOffsetResponse.Response.SafePruningOffset(offset) =>
-          Right(Some(UpstreamOffsetConvert.toParticipantOffset(offset)))
+          Right(Some(offset))
 
         case v30.GetSafePruningOffsetResponse.Response.NoSafePruningOffset(_) => Right(None)
       }
     }
 
-    final case class PruneInternallyCommand(pruneUpTo: ParticipantOffset)
+    final case class PruneInternallyCommand(pruneUpTo: String)
         extends Base[v30.PruneRequest, v30.PruneResponse, Unit] {
       override def createRequest(): Either[String, PruneRequest] =
-        pruneUpTo.value.absolute
-          .toRight("The pruneUpTo ledger offset needs to be absolute")
-          .map(v30.PruneRequest(_))
+        Right(v30.PruneRequest(pruneUpTo))
 
       override def submitRequest(
           service: PruningServiceStub,
@@ -1583,7 +1562,7 @@ object ParticipantAdminCommands {
     final case class SetNoWaitCommitmentsFrom(
         counterParticipants: Seq[ParticipantId],
         domainIds: Seq[DomainId],
-        startingAt: Either[Instant, ParticipantOffset],
+        startingAt: Either[Instant, String],
     ) extends Base[
           pruning.v30.SetNoWaitCommitmentsFrom.Request,
           pruning.v30.SetNoWaitCommitmentsFrom.Response,
@@ -1593,11 +1572,7 @@ object ParticipantAdminCommands {
         for {
           tsOrOffset <- startingAt match {
             case Right(offset) =>
-              offset.value match {
-                case Value.Absolute(value) =>
-                  Right(Right(value).withLeft[CantonTimestamp]).withLeft[String]
-                case other => Left(s"Unable to convert ledger_end `$other` to absolute value")
-              }
+              Right(Right(offset).withLeft[CantonTimestamp]).withLeft[String]
             case Left(ts) =>
               CantonTimestamp.fromInstant(ts) match {
                 case Left(value) => Left(value)
@@ -1643,7 +1618,7 @@ object ParticipantAdminCommands {
     // TODO(#18453) R6: The code below should be sufficient.
     final case class NoWaitCommitments(
         counterParticipant: ParticipantId,
-        startingAt: Either[Instant, ParticipantOffset],
+        startingAt: Either[Instant, String],
         domains: Seq[DomainId],
         state: SharedContractsState,
     )
@@ -1659,7 +1634,7 @@ object ParticipantAdminCommands {
             }
             offset <- setup.timestampOrOffsetActive match {
               case NoWaitCommitmentsSetup.TimestampOrOffsetActive.PruningOffset(offset) =>
-                Right(UpstreamOffsetConvert.toParticipantOffset(offset))
+                Right(offset)
               case _ => Left("Conversion error for Offset in ignoredParticipants")
             }
             domains <- setup.domainIds.traverse(_.domainIds.traverse(DomainId.fromString))
@@ -1673,7 +1648,7 @@ object ParticipantAdminCommands {
             participantId,
             setup.timestampOrOffsetActive match {
               case NoWaitCommitmentsSetup.TimestampOrOffsetActive.SequencingTimestamp(_) =>
-                ts.toInstant.asLeft[ParticipantOffset]
+                ts.toInstant.asLeft[String]
               case _ => offset.asRight[Instant]
             },
             domains.getOrElse(Seq.empty),

diff --git a/...main/scala/com/digitalasset/canton/admin/api/client/commands/SequencerAdminCommands.scala b/...main/scala/com/digitalasset/canton/admin/api/client/commands/SequencerAdminCommands.scala
@@ -15,7 +15,6 @@ import com.digitalasset.canton.admin.domain.v30.{
   SequencerStatusServiceGrpc,
 }
 import com.digitalasset.canton.config.RequireTypes.{NonNegativeLong, PositiveInt}
-import com.digitalasset.canton.data.CantonTimestamp
 import com.digitalasset.canton.domain.sequencing.sequencer.SequencerPruningStatus
 import com.digitalasset.canton.domain.sequencing.sequencer.traffic.TimestampSelector.TimestampSelector
 import com.digitalasset.canton.domain.sequencing.sequencer.traffic.{
@@ -100,7 +99,7 @@ object SequencerAdminCommands {
   ) extends BaseSequencerAdministrationCommands[
         admin.v30.SetTrafficPurchasedRequest,
         admin.v30.SetTrafficPurchasedResponse,
-        Option[CantonTimestamp],
+        Unit,
       ] {
     override def createRequest(): Either[String, admin.v30.SetTrafficPurchasedRequest] = Right(
       admin.v30.SetTrafficPurchasedRequest(member.toProtoPrimitive, serial.value, balance.value)
@@ -112,10 +111,7 @@ object SequencerAdminCommands {
       service.setTrafficPurchased(request)
     override def handleResponse(
         response: admin.v30.SetTrafficPurchasedResponse
-    ): Either[String, Option[CantonTimestamp]] =
-      response.maxSequencingTimestamp
-        .traverse(CantonTimestamp.fromProtoTimestamp)
-        .leftMap(_.message)
+    ): Either[String, Unit] = Right(())
   }
 
   object Health {