Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: extract retry config from resources and added default retry config #735

Open
wants to merge 17 commits into
base: main
Choose a base branch
from
Open

feat: extract retry config from resources and added default retry config #735

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
5313bf8
add 429 as an retryable status code
hqhqhqhqhqhqhqhqhqhqhq Dec 24, 2024
e42c144
update
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
22eb738
update
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
e00a32f
Update azapi_data_plane_resource.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
d7fc5da
update
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
f35993b
Update azapi_resource.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
a85c864
update
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
108022b
Update azapi_data_plane_resource.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
9ef5f6f
Update azapi_data_plane_resource.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
ff6455a
Update azapi_data_plane_resource.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
a6e8458
Update azapi_resource_action_ephemeral.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
6fbdad7
update
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
c6a456f
Update azapi_resource.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
ad6d6ac
Update azapi_data_plane_resource.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
4068e45
Update azapi_data_plane_resource.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
535f225
Update common.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
2caa155
Update common.go
hqhqhqhqhqhqhqhqhqhqhq Jan 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions internal/clients/data_plane_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"net/url"
"os"
"regexp"
"slices"
"strings"
Expand All @@ -18,6 +19,7 @@ import (
armruntime "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/runtime"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
"github.com/Azure/terraform-provider-azapi/internal/retry"
"github.com/Azure/terraform-provider-azapi/internal/services/parse"
"github.com/cenkalti/backoff/v4"
"github.com/hashicorp/terraform-plugin-log/tflog"
Expand Down Expand Up @@ -514,3 +516,43 @@ func isDataPlaneRetryable(ctx context.Context, retryclient DataPlaneClientRetrya
}
return false
}

func (retryclient *DataPlaneClient) ConfigureClientWithCustomRetry(ctx context.Context, retry retry.RetryValue) *DataPlaneClientRetryableErrors {
// configure default retry configuration
backOff := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(5*time.Second),
backoff.WithMaxInterval(30*time.Second),
backoff.WithMaxElapsedTime(RetryGet()),
)
errRegExps := retry.GetErrorMessagesRegex()
statusCodes := retry.GetDefaultRetryableStatusCodes()
dataCallbackFuncs := []func(d interface{}) bool{
func(d interface{}) bool {
return d == nil
},
}

if !retry.IsNull() && !retry.IsUnknown() {
tflog.Debug(ctx, "using custom retry configuration")
backOff = backoff.NewExponentialBackOff(
backoff.WithInitialInterval(retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(retry.GetMultiplier()),
backoff.WithRandomizationFactor(retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(RetryGet()),
)
errRegExps = StringSliceToRegexpSliceMust(retry.GetErrorMessages())
}

return retryclient.WithRetry(backOff, errRegExps, statusCodes, dataCallbackFuncs)
}

func RetryGet() time.Duration {
if v := os.Getenv("AZAPI_RETRY_GET_MAX_TIME"); v != "" {
timeout, err := time.ParseDuration(v)
if err == nil {
return timeout
}
}
return 2 * time.Minute
}
42 changes: 42 additions & 0 deletions internal/clients/resource_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"net/http"
"os"
"regexp"
"slices"
"strings"
Expand All @@ -16,6 +17,7 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
"github.com/Azure/terraform-provider-azapi/internal/retry"
"github.com/cenkalti/backoff/v4"
"github.com/hashicorp/terraform-plugin-log/tflog"
)
Expand Down Expand Up @@ -692,3 +694,43 @@ func isRetryable(ctx context.Context, retryclient ResourceClientRetryableErrors,
tflog.Debug(ctx, "isRetryable: Error is not retryable")
return false
}

func (client *ResourceClient) ConfigureClientWithCustomRetry(ctx context.Context, retry retry.RetryValue) Requester {
// configure default retry configuration
backOff := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(5*time.Second),
backoff.WithMaxInterval(30*time.Second),
backoff.WithMaxElapsedTime(RetryGetAfterPut()),
)
errRegExps := retry.GetErrorMessagesRegex()
statusCodes := retry.GetDefaultRetryableStatusCodes()
dataCallbackFuncs := []func(d interface{}) bool{
func(d interface{}) bool {
return d == nil
},
}

if !retry.IsNull() && !retry.IsUnknown() {
tflog.Debug(ctx, "using custom retry configuration")
backOff = backoff.NewExponentialBackOff(
backoff.WithInitialInterval(retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(retry.GetMultiplier()),
backoff.WithRandomizationFactor(retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(RetryGetAfterPut()),
)
errRegExps = StringSliceToRegexpSliceMust(retry.GetErrorMessages())
}

return client.WithRetry(backOff, errRegExps, statusCodes, dataCallbackFuncs)
}

func RetryGetAfterPut() time.Duration {
if v := os.Getenv("AZAPI_RETRY_GET_AFTER_PUT_MAX_TIME"); v != "" {
timeout, err := time.ParseDuration(v)
if err == nil {
return timeout
}
}
return 2 * time.Minute
}
6 changes: 6 additions & 0 deletions internal/retry/retryable_errors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
var (
_ basetypes.ObjectTypable = RetryType{}
_ basetypes.ObjectValuable = RetryValue{}

defaultRetryableStatusCodes = []int{404, 429}
)

const (
Expand Down Expand Up @@ -669,3 +671,7 @@ func (v RetryValue) AddDefaultValuesIfUnknownOrNull() RetryValue {
}
return v
}

func (v RetryValue) GetDefaultRetryableStatusCodes() []int {
return defaultRetryableStatusCodes
}
63 changes: 6 additions & 57 deletions internal/services/azapi_data_plane_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"github.com/Azure/terraform-provider-azapi/internal/services/parse"
"github.com/Azure/terraform-provider-azapi/internal/tf"
"github.com/Azure/terraform-provider-azapi/utils"
"github.com/cenkalti/backoff/v4"
"github.com/hashicorp/terraform-plugin-framework-timeouts/resource/timeouts"
"github.com/hashicorp/terraform-plugin-framework-validators/listvalidator"
"github.com/hashicorp/terraform-plugin-framework/attr"
Expand Down Expand Up @@ -390,20 +389,7 @@ func (r *DataPlaneResource) CreateUpdate(ctx context.Context, plan tfsdk.Plan, s
}
}

var client clients.DataPlaneRequester
client = r.ProviderData.DataPlaneClient
if !model.Retry.IsNull() {
regexps := clients.StringSliceToRegexpSliceMust(model.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(model.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(model.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(model.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(model.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(timeout),
)
tflog.Debug(ctx, "azapi_data_plane_resource.CreateUpdate is using retry")
client = r.ProviderData.DataPlaneClient.WithRetry(bkof, regexps, nil, nil)
}
client := r.ProviderData.DataPlaneClient.ConfigureClientWithCustomRetry(ctx, model.Retry)

ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
Expand Down Expand Up @@ -441,20 +427,8 @@ func (r *DataPlaneResource) CreateUpdate(ctx context.Context, plan tfsdk.Plan, s
}

// Create a new retry client to handle specific case of transient 404 after resource creation
clientGetAfterPut := r.ProviderData.DataPlaneClient.WithRetry(
backoff.NewExponentialBackOff(
backoff.WithInitialInterval(5*time.Second),
backoff.WithMaxInterval(30*time.Second),
backoff.WithMaxElapsedTime(RetryGetAfterPut()),
),
model.Retry.GetErrorMessagesRegex(),
[]int{404},
[]func(d interface{}) bool{
func(d interface{}) bool {
return d == nil
},
},
)
clientGetAfterPut := r.ProviderData.DataPlaneClient.ConfigureClientWithCustomRetry(ctx, retry.NewRetryValueNull())

responseBody, err := clientGetAfterPut.Get(ctx, id, clients.NewRequestOptions(AsMapOfString(model.ReadHeaders), AsMapOfLists(model.ReadQueryParameters)))
if err != nil {
if utils.ResponseErrorWasNotFound(err) {
Expand Down Expand Up @@ -500,20 +474,8 @@ func (r *DataPlaneResource) Read(ctx context.Context, request resource.ReadReque
}
ctx = tflog.SetField(ctx, "resource_id", id.ID())

var client clients.DataPlaneRequester
client = r.ProviderData.DataPlaneClient
if !model.Retry.IsNull() && !model.Retry.IsUnknown() {
regexps := clients.StringSliceToRegexpSliceMust(model.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(model.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(model.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(model.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(model.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(readTimeout),
)
tflog.Debug(ctx, "azapi_data_plane_resource.Read is using retry")
client = r.ProviderData.DataPlaneClient.WithRetry(bkof, regexps, nil, nil)
}
client := r.ProviderData.DataPlaneClient.ConfigureClientWithCustomRetry(ctx, model.Retry)

responseBody, err := client.Get(ctx, id, clients.NewRequestOptions(AsMapOfString(model.ReadHeaders), AsMapOfLists(model.ReadQueryParameters)))
if err != nil {
if utils.ResponseErrorWasNotFound(err) {
Expand Down Expand Up @@ -593,20 +555,7 @@ func (r *DataPlaneResource) Delete(ctx context.Context, request resource.DeleteR

ctx = tflog.SetField(ctx, "resource_id", id.ID())

var client clients.DataPlaneRequester
client = r.ProviderData.DataPlaneClient
if !model.Retry.IsNull() && !model.Retry.IsUnknown() {
regexps := clients.StringSliceToRegexpSliceMust(model.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(model.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(model.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(model.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(model.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(deleteTimeout),
)
tflog.Debug(ctx, "azapi_data_plane_resource.Delete is using retry")
client = r.ProviderData.DataPlaneClient.WithRetry(bkof, regexps, nil, nil)
}
client := r.ProviderData.DataPlaneClient.ConfigureClientWithCustomRetry(ctx, model.Retry)

lockIds := AsStringList(model.Locks)
slices.Sort(lockIds)
Expand Down
67 changes: 9 additions & 58 deletions internal/services/azapi_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (
"github.com/Azure/terraform-provider-azapi/internal/services/preflight"
"github.com/Azure/terraform-provider-azapi/internal/tf"
"github.com/Azure/terraform-provider-azapi/utils"
"github.com/cenkalti/backoff/v4"
"github.com/hashicorp/terraform-plugin-framework-timeouts/resource/timeouts"
"github.com/hashicorp/terraform-plugin-framework-validators/listvalidator"
"github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator"
Expand Down Expand Up @@ -622,20 +621,9 @@ func (r *AzapiResource) CreateUpdate(ctx context.Context, requestPlan tfsdk.Plan
return
}
}
var client clients.Requester
client = r.ProviderData.ResourceClient
if !plan.Retry.IsNull() {
regexps := clients.StringSliceToRegexpSliceMust(plan.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(plan.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(plan.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(plan.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(plan.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(timeout),
)
tflog.Debug(ctx, "azapi_resource.CreateUpdate is using retry")
client = r.ProviderData.ResourceClient.WithRetry(bkof, regexps, nil, nil)
}

client := r.ProviderData.ResourceClient.ConfigureClientWithCustomRetry(ctx, plan.Retry)

ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()

Expand Down Expand Up @@ -727,21 +715,10 @@ func (r *AzapiResource) CreateUpdate(ctx context.Context, requestPlan tfsdk.Plan
diagnostics.AddError("Failed to create/update resource", fmt.Errorf("creating/updating %s: %+v", id, err).Error())
return
}
// Create a new retry client to handle specific case of transient 404 or empty body after resource creation
clientGetAfterPut := r.ProviderData.ResourceClient.WithRetry(
backoff.NewExponentialBackOff(
backoff.WithInitialInterval(5*time.Second),
backoff.WithMaxInterval(30*time.Second),
backoff.WithMaxElapsedTime(RetryGetAfterPut()),
),
plan.Retry.GetErrorMessagesRegex(),
[]int{404},
[]func(d interface{}) bool{
func(d interface{}) bool {
return d == nil
},
},
)

// Create a new retry client to handle specific case of transient 404 or empty body after resource creation or other retryable errors
clientGetAfterPut := r.ProviderData.ResourceClient.ConfigureClientWithCustomRetry(ctx, plan.Retry)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The retry for the get after put is a special case and does not use the configuration in the resource's retry attribute. Removing the above code will cause a regression for resources deployed at MG scope, as this relies on the response check function:

			func(d interface{}) bool {
				return d == nil
			},


tflog.Debug(ctx, "azapi_resource.CreateUpdate get resource after creation")
responseBody, err := clientGetAfterPut.Get(ctx, id.AzureResourceId, id.ApiVersion, clients.NewRequestOptions(AsMapOfString(plan.ReadHeaders), AsMapOfLists(plan.ReadQueryParameters)))
if err != nil {
Expand Down Expand Up @@ -808,20 +785,7 @@ func (r *AzapiResource) Read(ctx context.Context, request resource.ReadRequest,

ctx = tflog.SetField(ctx, "resource_id", id.ID())

var client clients.Requester
client = r.ProviderData.ResourceClient
if !model.Retry.IsNull() && !model.Retry.IsUnknown() {
regexps := clients.StringSliceToRegexpSliceMust(model.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(model.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(model.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(model.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(model.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(readTimeout),
)
tflog.Debug(ctx, "azapi_resource.Read is using retry")
client = r.ProviderData.ResourceClient.WithRetry(bkof, regexps, nil, nil)
}
client := r.ProviderData.ResourceClient.ConfigureClientWithCustomRetry(ctx, model.Retry)

responseBody, err := client.Get(ctx, id.AzureResourceId, id.ApiVersion, clients.NewRequestOptions(AsMapOfString(model.ReadHeaders), AsMapOfLists(model.ReadQueryParameters)))
if err != nil {
Expand Down Expand Up @@ -956,20 +920,7 @@ func (r *AzapiResource) Delete(ctx context.Context, request resource.DeleteReque
return
}

var client clients.Requester
client = r.ProviderData.ResourceClient
if !model.Retry.IsNull() && !model.Retry.IsUnknown() {
regexps := clients.StringSliceToRegexpSliceMust(model.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(model.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(model.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(model.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(model.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(deleteTimeout),
)
tflog.Debug(ctx, "azapi_resource.Delete is using retry")
client = r.ProviderData.ResourceClient.WithRetry(bkof, regexps, nil, nil)
}
client := r.ProviderData.ResourceClient.ConfigureClientWithCustomRetry(ctx, model.Retry)

ctx, cancel := context.WithTimeout(ctx, deleteTimeout)
defer cancel()
Expand Down
16 changes: 1 addition & 15 deletions internal/services/azapi_resource_action_data_source.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"github.com/Azure/terraform-provider-azapi/internal/retry"
"github.com/Azure/terraform-provider-azapi/internal/services/myvalidator"
"github.com/Azure/terraform-provider-azapi/internal/services/parse"
"github.com/cenkalti/backoff/v4"
"github.com/hashicorp/terraform-plugin-framework-timeouts/resource/timeouts"
"github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator"
"github.com/hashicorp/terraform-plugin-framework/datasource"
Expand Down Expand Up @@ -183,20 +182,7 @@ func (r *ResourceActionDataSource) Read(ctx context.Context, request datasource.
method = "POST"
}

var client clients.Requester
client = r.ProviderData.ResourceClient
if !model.Retry.IsNull() && !model.Retry.IsUnknown() {
regexps := clients.StringSliceToRegexpSliceMust(model.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(model.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(model.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(model.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(model.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(readTimeout),
)
tflog.Debug(ctx, "data.azapi_resource_action.Read is using retry")
client = r.ProviderData.ResourceClient.WithRetry(bkof, regexps, nil, nil)
}
client := r.ProviderData.ResourceClient.ConfigureClientWithCustomRetry(ctx, model.Retry)

responseBody, err := client.Action(ctx, id.AzureResourceId, model.Action.ValueString(), id.ApiVersion, method, requestBody, clients.NewRequestOptions(AsMapOfString(model.Headers), AsMapOfLists(model.QueryParameters)))
if err != nil {
Expand Down
16 changes: 1 addition & 15 deletions internal/services/azapi_resource_action_ephemeral.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"github.com/Azure/terraform-provider-azapi/internal/retry"
"github.com/Azure/terraform-provider-azapi/internal/services/myvalidator"
"github.com/Azure/terraform-provider-azapi/internal/services/parse"
"github.com/cenkalti/backoff/v4"
"github.com/hashicorp/terraform-plugin-framework-timeouts/resource/timeouts"
"github.com/hashicorp/terraform-plugin-framework-validators/listvalidator"
"github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator"
Expand Down Expand Up @@ -189,20 +188,7 @@ func (r *ActionEphemeral) Open(ctx context.Context, request ephemeral.OpenReques
defer locks.UnlockByID(lockId)
}

var client clients.Requester
client = r.ProviderData.ResourceClient
if !model.Retry.IsNull() && !model.Retry.IsUnknown() {
regexps := clients.StringSliceToRegexpSliceMust(model.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(model.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(model.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(model.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(model.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(readTimeout),
)
tflog.Debug(ctx, "azapi_resource_action.Read is using retry")
client = r.ProviderData.ResourceClient.WithRetry(bkof, regexps, nil, nil)
}
client := r.ProviderData.ResourceClient.ConfigureClientWithCustomRetry(ctx, model.Retry)

responseBody, err := client.Action(ctx, id.AzureResourceId, model.Action.ValueString(), id.ApiVersion, method, requestBody, clients.NewRequestOptions(AsMapOfString(model.Headers), AsMapOfLists(model.QueryParameters)))
if err != nil {
Expand Down
Loading
Loading