-
Notifications
You must be signed in to change notification settings - Fork 900
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Worker Deployment Search Attributes (#7199)
## What changed? Add TemporalWorkerDeploymentVersion, TemporalWorkerDeployment, and TemporalWorkflowVersioningBehavior Search Attributes ## Why? To be able to list workflows based on their behavior, deployment, and most recently used deployment version. ## How did you test it? <!-- How have you verified this change? Tested locally? Added a unit test? Checked in staging env? --> ## Potential risks <!-- Assuming the worst case, what can be broken when deploying this change to production? --> ## Documentation <!-- Have you made sure this change doesn't falsify anything currently stated in `docs/`? If significant new behavior is added, have you described that in `docs/`? --> ## Is hotfix candidate? <!-- Is this PR a hotfix candidate or does it require a notification to be sent to the broader community? (Yes/No) --> --------- Co-authored-by: Shivam Saraf <[email protected]> Co-authored-by: Rodrigo Zhou <[email protected]>
- Loading branch information
3 people
authored
Feb 6, 2025
1 parent
2ade39c
commit 8b0d8f0
Showing
13 changed files
with
256 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| ./versioned/v8/index_template_v7.json | ||
| ./versioned/v9/index_template_v7.json |
111 changes: 111 additions & 0 deletions
111
schema/elasticsearch/visibility/versioned/v9/index_template_v7.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| { | ||
| "order": 0, | ||
| "index_patterns": ["temporal_visibility_v1*"], | ||
| "settings": { | ||
| "index": { | ||
| "number_of_shards": "1", | ||
| "number_of_replicas": "0", | ||
| "auto_expand_replicas": "0-2", | ||
| "search.idle.after": "365d", | ||
| "sort.field": ["CloseTime", "StartTime", "RunId"], | ||
| "sort.order": ["desc", "desc", "desc"], | ||
| "sort.missing": ["_first", "_first", "_first"] | ||
| } | ||
| }, | ||
| "mappings": { | ||
| "dynamic": "false", | ||
| "properties": { | ||
| "NamespaceId": { | ||
| "type": "keyword" | ||
| }, | ||
| "TemporalNamespaceDivision": { | ||
| "type": "keyword" | ||
| }, | ||
| "WorkflowId": { | ||
| "type": "keyword" | ||
| }, | ||
| "RunId": { | ||
| "type": "keyword" | ||
| }, | ||
| "WorkflowType": { | ||
| "type": "keyword" | ||
| }, | ||
| "StartTime": { | ||
| "type": "date_nanos" | ||
| }, | ||
| "ExecutionTime": { | ||
| "type": "date_nanos" | ||
| }, | ||
| "CloseTime": { | ||
| "type": "date_nanos" | ||
| }, | ||
| "ExecutionDuration": { | ||
| "type": "long" | ||
| }, | ||
| "ExecutionStatus": { | ||
| "type": "keyword" | ||
| }, | ||
| "TaskQueue": { | ||
| "type": "keyword" | ||
| }, | ||
| "TemporalChangeVersion": { | ||
| "type": "keyword" | ||
| }, | ||
| "BatcherNamespace": { | ||
| "type": "keyword" | ||
| }, | ||
| "BatcherUser": { | ||
| "type": "keyword" | ||
| }, | ||
| "BinaryChecksums": { | ||
| "type": "keyword" | ||
| }, | ||
| "HistoryLength": { | ||
| "type": "long" | ||
| }, | ||
| "StateTransitionCount": { | ||
| "type": "long" | ||
| }, | ||
| "TemporalScheduledStartTime": { | ||
| "type": "date_nanos" | ||
| }, | ||
| "TemporalScheduledById": { | ||
| "type": "keyword" | ||
| }, | ||
| "TemporalSchedulePaused": { | ||
| "type": "boolean" | ||
| }, | ||
| "HistorySizeBytes": { | ||
| "type": "long" | ||
| }, | ||
| "BuildIds": { | ||
| "type": "keyword" | ||
| }, | ||
| "ParentWorkflowId": { | ||
| "type": "keyword" | ||
| }, | ||
| "ParentRunId": { | ||
| "type": "keyword" | ||
| }, | ||
| "RootWorkflowId": { | ||
| "type": "keyword" | ||
| }, | ||
| "RootRunId": { | ||
| "type": "keyword" | ||
| }, | ||
| "TemporalPauseInfo": { | ||
| "type": "keyword" | ||
| }, | ||
| "TemporalWorkerDeploymentVersion": { | ||
| "type": "keyword" | ||
| }, | ||
| "TemporalWorkflowVersioningBehavior": { | ||
| "type": "keyword" | ||
| }, | ||
| "TemporalWorkerDeployment": { | ||
| "type": "keyword" | ||
| } | ||
| } | ||
| }, | ||
| "aliases": {} | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -eu -o pipefail | ||
|
|
||
| # Prerequisites: | ||
| # - jq | ||
| # - curl | ||
|
|
||
| # Input parameters. | ||
| : "${ES_SCHEME:=http}" | ||
| : "${ES_SERVER:=127.0.0.1}" | ||
| : "${ES_PORT:=9200}" | ||
| : "${ES_USER:=}" | ||
| : "${ES_PWD:=}" | ||
| : "${ES_VERSION:=v7}" | ||
| : "${ES_VIS_INDEX_V1:=temporal_visibility_v1_dev}" | ||
| : "${AUTO_CONFIRM:=}" | ||
| : "${SLICES_COUNT:=auto}" | ||
|
|
||
| es_endpoint="${ES_SCHEME}://${ES_SERVER}:${ES_PORT}" | ||
|
|
||
| echo "=== Step 0. Sanity check if Elasticsearch index is accessible ===" | ||
|
|
||
| if ! curl --silent --fail --user "${ES_USER}":"${ES_PWD}" "${es_endpoint}/${ES_VIS_INDEX_V1}/_stats/docs" --write-out "\n"; then | ||
| echo "Elasticsearch index ${ES_VIS_INDEX_V1} is not accessible at ${es_endpoint}." | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "=== Step 1. Add new builtin search attributes ===" | ||
|
|
||
| new_mapping=' | ||
| { | ||
| "properties": { | ||
| "TemporalWorkerDeploymentVersion": { | ||
| "type": "keyword" | ||
| }, | ||
| "TemporalWorkflowVersioningBehavior": { | ||
| "type": "keyword" | ||
| }, | ||
| "TemporalWorkerDeployment": { | ||
| "type": "keyword" | ||
| } | ||
| } | ||
| } | ||
| ' | ||
|
|
||
| if [ -z "${AUTO_CONFIRM}" ]; then | ||
| read -p "Add new builtin search attributes to the index ${ES_VIS_INDEX_V1}? (N/y)" -n 1 -r | ||
| echo | ||
| else | ||
| REPLY="y" | ||
| fi | ||
| if [ "${REPLY}" = "y" ]; then | ||
| curl --silent --fail --user "${ES_USER}":"${ES_PWD}" -X PUT "${es_endpoint}/${ES_VIS_INDEX_V1}/_mapping" -H "Content-Type: application/json" --data-binary "$new_mapping" | jq | ||
| # Wait for mapping changes to go through. | ||
| until curl --silent --user "${ES_USER}":"${ES_PWD}" "${es_endpoint}/_cluster/health/${ES_VIS_INDEX_V1}" | jq --exit-status '.status=="green" | .'; do | ||
| echo "Waiting for Elasticsearch index ${ES_VIS_INDEX_V1} become green." | ||
| sleep 1 | ||
| done | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
7 changes: 7 additions & 0 deletions
7
schema/mysql/v8/visibility/versioned/v1.8/add_deployment_search_attributes.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| ALTER TABLE executions_visibility ADD COLUMN TemporalWorkerDeploymentVersion VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalWorkerDeploymentVersion"); | ||
| ALTER TABLE executions_visibility ADD COLUMN TemporalWorkflowVersioningBehavior VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalWorkflowVersioningBehavior"); | ||
| ALTER TABLE executions_visibility ADD COLUMN TemporalWorkerDeployment VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalWorkerDeployment"); | ||
|
|
||
| CREATE INDEX by_temporal_worker_deployment_version ON executions_visibility (namespace_id, TemporalWorkerDeploymentVersion, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); | ||
| CREATE INDEX by_temporal_workflow_versioning_behavior ON executions_visibility (namespace_id, TemporalWorkflowVersioningBehavior, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); | ||
| CREATE INDEX by_temporal_worker_deployment ON executions_visibility (namespace_id, TemporalWorkerDeployment, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| { | ||
| "CurrVersion": "1.8", | ||
| "MinCompatibleVersion": "0.1", | ||
| "Description": "add TemporalWorkerDeploymentVersion, TemporalWorkerDeployment, and TemporalWorkflowVersioningBehavior columns", | ||
| "SchemaUpdateCqlFiles": [ | ||
| "add_deployment_search_attributes.sql" | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
7 changes: 7 additions & 0 deletions
7
schema/postgresql/v12/visibility/versioned/v1.8/add_deployment_search_attributes.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| ALTER TABLE executions_visibility ADD COLUMN TemporalWorkerDeploymentVersion VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>'TemporalWorkerDeploymentVersion') STORED; | ||
| ALTER TABLE executions_visibility ADD COLUMN TemporalWorkflowVersioningBehavior VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>'TemporalWorkflowVersioningBehavior') STORED; | ||
| ALTER TABLE executions_visibility ADD COLUMN TemporalWorkerDeployment VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>'TemporalWorkerDeployment') STORED; | ||
|
|
||
| CREATE INDEX by_temporal_worker_deployment_version ON executions_visibility (namespace_id, TemporalWorkerDeploymentVersion, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); | ||
| CREATE INDEX by_temporal_workflow_versioning_behavior ON executions_visibility (namespace_id, TemporalWorkflowVersioningBehavior, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); | ||
| CREATE INDEX by_temporal_worker_deployment ON executions_visibility (namespace_id, TemporalWorkerDeployment, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); |
8 changes: 8 additions & 0 deletions
8
schema/postgresql/v12/visibility/versioned/v1.8/manifest.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| { | ||
| "CurrVersion": "1.8", | ||
| "MinCompatibleVersion": "0.1", | ||
| "Description": "add TemporalWorkerDeploymentVersion, TemporalWorkerDeployment, and TemporalWorkflowVersioningBehavior columns", | ||
| "SchemaUpdateCqlFiles": [ | ||
| "add_deployment_search_attributes.sql" | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters