[main] Update 2024-10-15.23 (#239)

Reference commit: 8fd14bea37

Co-authored-by: Canton <[email protected]>

UNRELEASED.md

@@ -3,6 +3,12 @@
 Canton CANTON_VERSION has been released on RELEASE_DATE. You can download the Daml Open Source edition from the Daml Connect [Github Release Section](https://github.com/digital-asset/daml/releases/tag/vCANTON_VERSION). The Enterprise edition is available on [Artifactory](https://digitalasset.jfrog.io/artifactory/canton-enterprise/canton-enterprise-CANTON_VERSION.zip).
 Please also consult the [full documentation of this release](https://docs.daml.com/CANTON_VERSION/canton/about.html).
 
+## Until 2024-10-16 (Exclusive)
+
+- New config option `parameters.timeouts.processing.sequenced-event-processing-bound` allows to specify a timeout for processing sequenced events. When processing takes longer on a node, the node will log an error or crash (depending on the `exit-on-fatal-failures` parameter).
+- Fixed a crash recovery bug in unified sequencer, when it can miss events in the recovery process. Now it will start from
+  the correct earlier block height in these situations.
+
 ## Until 2024-10-02 (Exclusive)
 
 - Removed party-level group addressing.
@@ -30,6 +36,7 @@ canton {
       }
       block {
         writer.checkpoint-interval = "10s"
+        checkpoint-backfill-parallelism = 2
         reader.read-batch-size = 50
       }
     }
@@ -56,6 +63,10 @@ The integer approach replaces string representation in:
 - GetLatestPrunedOffsetsResponse message: with optional int64
   - If specified, it is a valid absolute offset (positive integer).
   - If not set, no pruning has happened yet.
+- SubmitAndWaitForUpdateIdResponse message: with int64
+- PruneRequest message (prune_up_to): with int64
+- Reassignment, TransactionTree, Transaction and Completion (offset, deduplication_offset) message: with int64
+- Commands message (deduplication_offset): with int64
 
 ## Until 2024-09-16 (Exclusive)
 

community/app-base/src/main/scala/com/digitalasset/canton/admin/api/client/GrpcCtlRunner.scala

@@ -6,6 +6,7 @@ package com.digitalasset.canton.admin.api.client
 import cats.data.EitherT
 import com.daml.grpc.AuthCallCredentials
 import com.digitalasset.canton.admin.api.client.commands.GrpcAdminCommand
+import com.digitalasset.canton.lifecycle.OnShutdownRunner
 import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging}
 import com.digitalasset.canton.networking.grpc.CantonGrpcUtil
 import com.digitalasset.canton.tracing.{TraceContext, TraceContextGrpc}
@@ -21,6 +22,7 @@ import scala.concurrent.{ExecutionContext, Future}
 class GrpcCtlRunner(
     maxRequestDebugLines: Int,
     maxRequestDebugStringLength: Int,
+    onShutdownRunner: OnShutdownRunner,
     val loggerFactory: NamedLoggerFactory,
 ) extends NamedLogging {
 
@@ -53,7 +55,7 @@ class GrpcCtlRunner(
   )(instanceName: String, service: command.Svc, request: Req, timeout: Duration)(implicit
       ec: ExecutionContext,
       traceContext: TraceContext,
-  ): EitherT[Future, String, Res] =
+  ): EitherT[Future, String, Res] = CantonGrpcUtil.shutdownAsGrpcErrorE(
     CantonGrpcUtil
       .sendGrpcRequest(service, instanceName)(
         command.submitRequest(_, request),
@@ -62,8 +64,10 @@ class GrpcCtlRunner(
         ),
         timeout,
         logger,
+        onShutdownRunner,
         CantonGrpcUtil.silentLogPolicy, // silent log policy, as the ConsoleEnvironment will log the result
         _ => false, // no retry to optimize for low latency
       )
       .leftMap(_.toString)
+  )
 }

community/app-base/src/main/scala/com/digitalasset/canton/admin/api/client/commands/LedgerApiCommands.scala

@@ -12,21 +12,21 @@ import com.daml.ledger.api.v2.admin.command_inspection_service.{
   GetCommandStatusRequest,
   GetCommandStatusResponse,
 }
-import com.daml.ledger.api.v2.admin.identity_provider_config_service.IdentityProviderConfigServiceGrpc.IdentityProviderConfigServiceStub
 import com.daml.ledger.api.v2.admin.identity_provider_config_service.*
+import com.daml.ledger.api.v2.admin.identity_provider_config_service.IdentityProviderConfigServiceGrpc.IdentityProviderConfigServiceStub
 import com.daml.ledger.api.v2.admin.metering_report_service.MeteringReportServiceGrpc.MeteringReportServiceStub
 import com.daml.ledger.api.v2.admin.metering_report_service.{
   GetMeteringReportRequest,
   GetMeteringReportResponse,
   MeteringReportServiceGrpc,
 }
 import com.daml.ledger.api.v2.admin.object_meta.ObjectMeta
-import com.daml.ledger.api.v2.admin.package_management_service.PackageManagementServiceGrpc.PackageManagementServiceStub
 import com.daml.ledger.api.v2.admin.package_management_service.*
-import com.daml.ledger.api.v2.admin.participant_pruning_service.ParticipantPruningServiceGrpc.ParticipantPruningServiceStub
+import com.daml.ledger.api.v2.admin.package_management_service.PackageManagementServiceGrpc.PackageManagementServiceStub
 import com.daml.ledger.api.v2.admin.participant_pruning_service.*
-import com.daml.ledger.api.v2.admin.party_management_service.PartyManagementServiceGrpc.PartyManagementServiceStub
+import com.daml.ledger.api.v2.admin.participant_pruning_service.ParticipantPruningServiceGrpc.ParticipantPruningServiceStub
 import com.daml.ledger.api.v2.admin.party_management_service.*
+import com.daml.ledger.api.v2.admin.party_management_service.PartyManagementServiceGrpc.PartyManagementServiceStub
 import com.daml.ledger.api.v2.admin.user_management_service.UserManagementServiceGrpc.UserManagementServiceStub
 import com.daml.ledger.api.v2.admin.user_management_service.{
   CreateUserRequest,
@@ -437,7 +437,7 @@ object LedgerApiCommands {
       override def createRequest(): Either[String, PruneRequest] =
         Right(
           PruneRequest(
-            pruneUpTo,
+            ApiOffset.assertFromStringToLong(pruneUpTo),
             // canton always prunes divulged contracts both in the ledger api index-db and in canton stores
             pruneAllDivulgedContracts = true,
           )
@@ -998,6 +998,8 @@ object LedgerApiCommands {
           case u: UpdateService.AssignedWrapper => u.assignedEvent.createdEvent
           case _: UpdateService.UnassignedWrapper => None
         }
+
+      def domainId: String
     }
     object UpdateWrapper {
       def isUnassignedWrapper(wrapper: UpdateWrapper): Boolean = wrapper match {
@@ -1013,11 +1015,12 @@ object LedgerApiCommands {
       override def updateId: String = transaction.updateId
       override def isUnassignment: Boolean = false
 
+      override def domainId: String = transaction.domainId
     }
     sealed trait ReassignmentWrapper extends UpdateTreeWrapper with UpdateWrapper {
       def reassignment: Reassignment
       def unassignId: String = reassignment.getUnassignedEvent.unassignId
-      def offset: String = reassignment.offset
+      def offset: String = ApiOffset.fromLong(reassignment.offset)
     }
     object ReassignmentWrapper {
       def apply(reassignment: Reassignment): ReassignmentWrapper = {
@@ -1038,11 +1041,16 @@ object LedgerApiCommands {
         extends ReassignmentWrapper {
       override def updateId: String = reassignment.updateId
       override def isUnassignment: Boolean = false
+
+      override def domainId: String = assignedEvent.target
     }
     final case class UnassignedWrapper(reassignment: Reassignment, unassignedEvent: UnassignedEvent)
         extends ReassignmentWrapper {
       override def updateId: String = reassignment.updateId
       override def isUnassignment: Boolean = true
+
+      override def domainId: String = unassignedEvent.source
+
     }
 
     trait BaseCommand[Req, Resp, Res] extends GrpcAdminCommand[Req, Resp, Res] {
@@ -1187,7 +1195,7 @@ object LedgerApiCommands {
           )
         case DeduplicationPeriod.DeduplicationOffset(offset) =>
           Commands.DeduplicationPeriod.DeduplicationOffset(
-            offset.toHexString
+            offset.toLong
           )
       },
       minLedgerTimeAbs = minLedgerTimeAbs.map(ProtoConverter.InstantConverter.toProtoPrimitive),
@@ -1232,9 +1240,13 @@ object LedgerApiCommands {
         override val packageIdSelectionPreference: Seq[LfPackageId],
     ) extends SubmitCommand
         with BaseCommand[SubmitRequest, SubmitResponse, Unit] {
-      override def createRequest(): Either[String, SubmitRequest] = Right(
-        SubmitRequest(commands = Some(mkCommand))
-      )
+      override def createRequest(): Either[String, SubmitRequest] =
+        try {
+          Right(SubmitRequest(commands = Some(mkCommand)))
+        } catch {
+          case t: Throwable =>
+            Left(t.getMessage)
+        }
 
       override def submitRequest(
           service: CommandSubmissionServiceStub,
@@ -1419,7 +1431,7 @@ object LedgerApiCommands {
           )
         case DeduplicationPeriod.DeduplicationOffset(offset) =>
           ExecuteSubmissionRequest.DeduplicationPeriod.DeduplicationOffset(
-            offset.toHexString
+            offset.toLong
           )
       }
 
@@ -1479,7 +1491,12 @@ object LedgerApiCommands {
         ] {
 
       override def createRequest(): Either[String, SubmitAndWaitRequest] =
-        Right(SubmitAndWaitRequest(commands = Some(mkCommand)))
+        try {
+          Right(SubmitAndWaitRequest(commands = Some(mkCommand)))
+        } catch {
+          case t: Throwable =>
+            Left(t.getMessage)
+        }
 
       override def submitRequest(
           service: CommandServiceStub,
@@ -1513,7 +1530,12 @@ object LedgerApiCommands {
         with BaseCommand[SubmitAndWaitRequest, SubmitAndWaitForTransactionResponse, Transaction] {
 
       override def createRequest(): Either[String, SubmitAndWaitRequest] =
-        Right(SubmitAndWaitRequest(commands = Some(mkCommand)))
+        try {
+          Right(SubmitAndWaitRequest(commands = Some(mkCommand)))
+        } catch {
+          case t: Throwable =>
+            Left(t.getMessage)
+        }
 
       override def submitRequest(
           service: CommandServiceStub,

community/app-base/src/main/scala/com/digitalasset/canton/admin/api/client/commands/SequencerPublicCommands.scala

@@ -5,9 +5,9 @@ package com.digitalasset.canton.admin.api.client.commands
 
 import cats.syntax.either.*
 import com.digitalasset.canton.admin.api.client.data.StaticDomainParameters as ConsoleStaticDomainParameters
+import com.digitalasset.canton.domain.api.v30 as proto
 import com.digitalasset.canton.domain.api.v30.SequencerConnect.GetDomainParametersResponse.Parameters
 import com.digitalasset.canton.domain.api.v30.SequencerConnectServiceGrpc.SequencerConnectServiceStub
-import com.digitalasset.canton.domain.api.v30 as proto
 import com.digitalasset.canton.topology.DomainId
 import com.google.protobuf.empty.Empty
 import io.grpc.ManagedChannel

community/app-base/src/main/scala/com/digitalasset/canton/admin/api/client/commands/TopologyAdminCommands.scala

@@ -18,12 +18,12 @@ import com.digitalasset.canton.topology.*
 import com.digitalasset.canton.topology.admin.grpc.BaseQuery
 import com.digitalasset.canton.topology.admin.grpc.TopologyStore.Domain
 import com.digitalasset.canton.topology.admin.v30
+import com.digitalasset.canton.topology.admin.v30.*
 import com.digitalasset.canton.topology.admin.v30.AuthorizeRequest.Type.{Proposal, TransactionHash}
 import com.digitalasset.canton.topology.admin.v30.IdentityInitializationServiceGrpc.IdentityInitializationServiceStub
 import com.digitalasset.canton.topology.admin.v30.TopologyAggregationServiceGrpc.TopologyAggregationServiceStub
 import com.digitalasset.canton.topology.admin.v30.TopologyManagerReadServiceGrpc.TopologyManagerReadServiceStub
 import com.digitalasset.canton.topology.admin.v30.TopologyManagerWriteServiceGrpc.TopologyManagerWriteServiceStub
-import com.digitalasset.canton.topology.admin.v30.*
 import com.digitalasset.canton.topology.store.StoredTopologyTransactions
 import com.digitalasset.canton.topology.store.StoredTopologyTransactions.GenericStoredTopologyTransactions
 import com.digitalasset.canton.topology.transaction.SignedTopologyTransaction.GenericSignedTopologyTransaction
@@ -583,7 +583,7 @@ object TopologyAdminCommands {
         domainStore.flatMap(domainId =>
           Right(
             v30.GenesisStateRequest(
-              domainId.map(Domain).map(_.toProto),
+              domainId.map(Domain.apply).map(_.toProto),
               timestamp.map(_.toProtoTimestamp),
             )
           )

community/app-base/src/main/scala/com/digitalasset/canton/admin/api/client/data/ComponentStatus.scala

@@ -20,8 +20,6 @@ import com.digitalasset.canton.util.ShowUtil
 import io.circe.Encoder
 import io.circe.generic.semiauto.deriveEncoder
 
-import scala.annotation.nowarn
-
 final case class ComponentStatus(name: String, state: ComponentHealthState) extends PrettyPrinting {
   override val pretty: Pretty[ComponentStatus] = ComponentStatus.componentStatusPretty
 }
@@ -55,7 +53,6 @@ object ComponentStatus {
         ).asRight
     }
 
-  @nowarn("cat=lint-byname-implicit") // https://github.com/scala/bug/issues/12072
   implicit val componentStatusEncoder: Encoder[ComponentStatus] = deriveEncoder[ComponentStatus]
 
   implicit val componentStatusPretty: Pretty[ComponentStatus] = {
@@ -79,7 +76,6 @@ object ComponentHealthState extends ShowUtil {
   private val NotInitializedState: ComponentHealthState =
     ComponentHealthState.failed("Not Initialized")
 
-  @nowarn("cat=lint-byname-implicit") // https://github.com/scala/bug/issues/12072
   implicit val componentHealthStateEncoder: Encoder[ComponentHealthState] =
     deriveEncoder[ComponentHealthState]
 
@@ -109,7 +105,6 @@ object ComponentHealthState extends ShowUtil {
       with HasUnhealthyState {}
 
   object Degraded {
-    @nowarn("cat=lint-byname-implicit") // https://github.com/scala/bug/issues/12072
     implicit val degradedEncoder: Encoder[Degraded] = deriveEncoder[Degraded]
   }
 
@@ -122,7 +117,6 @@ object ComponentHealthState extends ShowUtil {
       with HasUnhealthyState {}
 
   object Failed {
-    @nowarn("cat=lint-byname-implicit") // https://github.com/scala/bug/issues/12072
     implicit val failedEncoder: Encoder[Failed] = deriveEncoder[Failed]
   }