From 5c474ddcdcd30045ef7152c0b5bf04e0a0fd2737 Mon Sep 17 00:00:00 2001 From: Hannah Hunter Date: Wed, 10 Jul 2024 11:52:35 -0400 Subject: [PATCH 1/5] add section for overriding db saved message Signed-off-by: Hannah Hunter --- .../state-management/howto-outbox.md | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md b/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md index 59dcf3c3711..9f995eaed79 100644 --- a/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md +++ b/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md @@ -34,7 +34,7 @@ The outbox feature can be used with using any [transactional state store]({{< re Message brokers that work with the competing consumer pattern (for example, [Apache Kafka]({{< ref setup-apache-kafka>}})) are encouraged to reduce the chances of duplicate events. {{% /alert %}} -## Usage +## Enable the outbox pattern To enable the outbox feature, add the following required and optional fields on a state store component: @@ -68,6 +68,8 @@ spec: | outboxPubsub | No | `outboxPublishPubsub` | Sets the pub/sub component used by Dapr to coordinate the state and pub/sub transactions. If not set, the pub/sub component configured with `outboxPublishPubsub` is used. This is useful if you want to separate the pub/sub component used to send the notification state changes from the one used to coordinate the transaction | outboxDiscardWhenMissingState | No | `false` | By setting `outboxDiscardWhenMissingState` to `true`, Dapr discards the transaction if it cannot find the state in the database and does not retry. This setting can be useful if the state store data has been deleted for any reason before Dapr was able to deliver the message and you would like Dapr to drop the items from the pub/sub and stop retrying to fetch the state +## Additional configurations + ### Combining outbox and non-outbox messages on the same state store If you want to use the same state store for sending both outbox and non-outbox messages, simply define two state store components that connect to the same state store, where one has the outbox feature and the other does not. @@ -106,6 +108,40 @@ spec: value: "newOrder" ``` +### Shape the outbox pattern message + +You can override the outbox pattern message saved to the database during the transaction by setting a different message. This is done via a projected transaction payload, which is ignored in the database, but used as the outbox pattern message published to the user topic. + +In the following Go example of a state transaction, the value of `"2"` is saved to the database, but the value of `"3"` is published to the end-user topic. + +```go +_, err = runtimev1pb.NewDaprClient(conn).ExecuteStateTransaction(ctx, &runtimev1pb.ExecuteStateTransactionRequest{ + StoreName: "mystore", + Operations: []*runtimev1pb.TransactionalStateOperation{ + { + OperationType: "upsert", + Request: &common.StateItem{ + Key: "1", + Value: []byte("2"), + }, + }, + { + OperationType: "upsert", + Request: &common.StateItem{ + Key: "1", + Value: []byte("3"), + // Override the data payload saved to the database + Metadata: map[string]string{ + "outbox.projection": "true", + }, + }, + }, + }, + }) +``` + +By setting the metadata item `"outbox.projection"` to `"true"`, the transaction value saved to the database is ignored, while the second value is published to the configured pub/sub topic. + ## Demo Watch [this video for an overview of the outbox pattern](https://youtu.be/rTovKpG0rhY?t=1338): From eb69cd916ecd5697751ec140f1e852e93aefb4fd Mon Sep 17 00:00:00 2001 From: Hannah Hunter Date: Fri, 12 Jul 2024 15:10:29 -0400 Subject: [PATCH 2/5] update example based on go sdk for transactional state Signed-off-by: Hannah Hunter --- .../state-management/howto-outbox.md | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md b/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md index 9f995eaed79..7b3e25b1c02 100644 --- a/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md +++ b/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md @@ -115,29 +115,29 @@ You can override the outbox pattern message saved to the database during the tra In the following Go example of a state transaction, the value of `"2"` is saved to the database, but the value of `"3"` is published to the end-user topic. ```go -_, err = runtimev1pb.NewDaprClient(conn).ExecuteStateTransaction(ctx, &runtimev1pb.ExecuteStateTransactionRequest{ - StoreName: "mystore", - Operations: []*runtimev1pb.TransactionalStateOperation{ - { - OperationType: "upsert", - Request: &common.StateItem{ - Key: "1", - Value: []byte("2"), - }, - }, - { - OperationType: "upsert", - Request: &common.StateItem{ - Key: "1", - Value: []byte("3"), - // Override the data payload saved to the database - Metadata: map[string]string{ - "outbox.projection": "true", - }, - }, - }, - }, - }) +ops := make([]*dapr.StateOperation, 0) + +op1 := &dapr.StateOperation{ + Type: dapr.StateOperationTypeUpsert, + Item: &dapr.SetStateItem{ + Key: "key1", + Value: []byte("2"), + }, +} +op2 := &dapr.StateOperation{ + Type: dapr.StateOperationTypeUpsert, + Item: &dapr.SetStateItem{ + Key: "key1", + Value: []byte("3"), + // Override the data payload saved to the database + Metadata: map[string]string{ + "outbox.projection": "true", + }, + }, +} +ops = append(ops, op1, op2) +meta := map[string]string{} +err := testClient.ExecuteStateTransaction(ctx, store, meta, ops) ``` By setting the metadata item `"outbox.projection"` to `"true"`, the transaction value saved to the database is ignored, while the second value is published to the configured pub/sub topic. From 8b105e9949538391c9065f53a2d77ef31129c9ef Mon Sep 17 00:00:00 2001 From: Hannah Hunter Date: Tue, 16 Jul 2024 10:59:07 -0400 Subject: [PATCH 3/5] add section for overriding cloudevent fields Signed-off-by: Hannah Hunter --- .../state-management/howto-outbox.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md b/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md index 7b3e25b1c02..64aca99607e 100644 --- a/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md +++ b/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md @@ -142,6 +142,39 @@ err := testClient.ExecuteStateTransaction(ctx, store, meta, ops) By setting the metadata item `"outbox.projection"` to `"true"`, the transaction value saved to the database is ignored, while the second value is published to the configured pub/sub topic. +### Override Dapr-generated cloudevent fields + +You can also override the [Dapr-generated cloudevent fields]({{< ref "pubsub-cloudevents.md#dapr-generated-cloudevents-example" >}}) on the published outbox event with custom cloudevent metadata. + +```go +ops := make([]*dapr.StateOperation, 0) + +op1 := &dapr.StateOperation{ + Type: dapr.StateOperationTypeUpsert, + Item: &dapr.SetStateItem{ + Key: "key1", + Value: []byte("2"), + // Override the data payload saved to the database + Metadata: map[string]string{ + "outbox.projection": "true", + "outbox.cloudevent.id": "unique-business-process-id", + "outbox.cloudevent.source": "CustomersApp", + "outbox.cloudevent.type": "CustomerCreated", + "outbox.cloudevent.subject": "123", + "outbox.cloudevent.my-custom-ce-field": "abc", + }, + }, +} +ops = append(ops, op1, op2) +meta := map[string]string{} +err := testClient.ExecuteStateTransaction(ctx, store, meta, ops) +``` + +{{% alert title="Note" color="primary" %}} +The `outbox.cloudevent.data` metadata is reserved for Dapr's use only, and is non-customizable. + +{{% /alert %}} + ## Demo Watch [this video for an overview of the outbox pattern](https://youtu.be/rTovKpG0rhY?t=1338): From 138b777b697cd0bbb087d34e3cce52b9e281267c Mon Sep 17 00:00:00 2001 From: Hannah Hunter Date: Tue, 16 Jul 2024 12:46:46 -0400 Subject: [PATCH 4/5] add http tabs and remove outbox.cloudevent.* format Signed-off-by: Hannah Hunter --- .../state-management/howto-outbox.md | 108 ++++++++++++++++-- 1 file changed, 96 insertions(+), 12 deletions(-) diff --git a/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md b/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md index 64aca99607e..9608279007b 100644 --- a/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md +++ b/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md @@ -110,9 +110,15 @@ spec: ### Shape the outbox pattern message -You can override the outbox pattern message saved to the database during the transaction by setting a different message. This is done via a projected transaction payload, which is ignored in the database, but used as the outbox pattern message published to the user topic. +You can override the outbox pattern message published to the pub/sub broker by setting a different message. This is done via a projected transaction payload, which is ignored, but used as the outbox pattern message published to the user topic. -In the following Go example of a state transaction, the value of `"2"` is saved to the database, but the value of `"3"` is published to the end-user topic. +{{< tabs "Go SDK" HTTP >}} + +{{% codetab %}} + + + +In the following Go SDK example of a state transaction, the value of `"2"` is saved to the database, but the value of `"3"` is published to the end-user topic. ```go ops := make([]*dapr.StateOperation, 0) @@ -140,11 +146,57 @@ meta := map[string]string{} err := testClient.ExecuteStateTransaction(ctx, store, meta, ops) ``` -By setting the metadata item `"outbox.projection"` to `"true"`, the transaction value saved to the database is ignored, while the second value is published to the configured pub/sub topic. +By setting the metadata item `"outbox.projection"` to `"true"`, the first transaction value published to the broker is ignored, while the second value is published to the configured pub/sub topic. + +{{% /codetab %}} + +{{% codetab %}} + + + +You can pass the message override using the following HTTP request: + +```bash +curl -X POST http://localhost:3500/v1.0/state/starwars/transaction \ + -H "Content-Type: application/json" \ + -d '{ + "operations": [ + { + "operation": "upsert", + "request": { + "key": "key1", + "value": "2" + } + }, + { + "operation": "upsert", + "request": { + "key": "key1" + "value: "3" + "metadata": { + "outboxProjection": "true" + } + } + } + ], + }' +``` + +By setting the metadata item `"outboxProjection"` to `"true"`, the first transaction value published to the broker is ignored, while the second value is published to the configured pub/sub topic. + +{{% /codetab %}} + +{{< /tabs >}} -### Override Dapr-generated cloudevent fields +### Override Dapr-generated CloudEvent fields -You can also override the [Dapr-generated cloudevent fields]({{< ref "pubsub-cloudevents.md#dapr-generated-cloudevents-example" >}}) on the published outbox event with custom cloudevent metadata. +You can also override the [Dapr-generated CloudEvent fields]({{< ref "pubsub-cloudevents.md#dapr-generated-cloudevents-example" >}}) on the published outbox event with custom CloudEvent metadata. + +{{< tabs "Go SDK" HTTP >}} + +{{% codetab %}} + + ```go ops := make([]*dapr.StateOperation, 0) @@ -156,12 +208,11 @@ op1 := &dapr.StateOperation{ Value: []byte("2"), // Override the data payload saved to the database Metadata: map[string]string{ - "outbox.projection": "true", - "outbox.cloudevent.id": "unique-business-process-id", - "outbox.cloudevent.source": "CustomersApp", - "outbox.cloudevent.type": "CustomerCreated", - "outbox.cloudevent.subject": "123", - "outbox.cloudevent.my-custom-ce-field": "abc", + "id": "unique-business-process-id", + "source": "CustomersApp", + "type": "CustomerCreated", + "subject": "123", + "my-custom-ce-field": "abc", }, }, } @@ -169,9 +220,42 @@ ops = append(ops, op1, op2) meta := map[string]string{} err := testClient.ExecuteStateTransaction(ctx, store, meta, ops) ``` +{{% /codetab %}} + +{{% codetab %}} + + + +```bash +curl -X POST http://localhost:3500/v1.0/state/starwars/transaction \ + -H "Content-Type: application/json" \ + -d '{ + "operations": [ + { + "operation": "upsert", + "request": { + "key": "key1", + "value": "2" + } + }, + ], + "metadata": { + "id": "unique-business-process-id", + "source": "CustomersApp", + "type": "CustomerCreated", + "subject": "123", + "my-custom-ce-field": "abc", + } + }' +``` + +{{% /codetab %}} + +{{< /tabs >}} + {{% alert title="Note" color="primary" %}} -The `outbox.cloudevent.data` metadata is reserved for Dapr's use only, and is non-customizable. +The `data` CloudEvent field is reserved for Dapr's use only, and is non-customizable. {{% /alert %}} From 6ffd987f539eebe4c57edf00f8ba5d13715f0274 Mon Sep 17 00:00:00 2001 From: Yaron Schneider Date: Tue, 16 Jul 2024 09:58:49 -0700 Subject: [PATCH 5/5] remove redundant space --- .../building-blocks/state-management/howto-outbox.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md b/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md index 9608279007b..a78282255d1 100644 --- a/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md +++ b/daprdocs/content/en/developing-applications/building-blocks/state-management/howto-outbox.md @@ -146,7 +146,7 @@ meta := map[string]string{} err := testClient.ExecuteStateTransaction(ctx, store, meta, ops) ``` -By setting the metadata item `"outbox.projection"` to `"true"`, the first transaction value published to the broker is ignored, while the second value is published to the configured pub/sub topic. +By setting the metadata item `"outbox.projection"` to `"true"`, the first transaction value published to the broker is ignored, while the second value is published to the configured pub/sub topic. {{% /codetab %}}