Skip to content

Commit

Permalink
retry calls on http2 goaway frames (linode#286)
Browse files Browse the repository at this point in the history
## 📝 Description

**What does this PR do and why is this change necessary?**

whenever resty receives a goaway frame retry.

## ✔️ How to Test

**What are the steps to reproduce the issue or verify the changes?**

`make ARGS='-run TestClient_GoAwayRetry' fixtures`

resolves linode#283 

Co-authored-by: Lena Garber <[email protected]>
  • Loading branch information
jriddle-linode and lgarber-akamai authored Dec 22, 2022
1 parent 089fc9f commit 5451945
Show file tree
Hide file tree
Showing 23 changed files with 513 additions and 1,127 deletions.
1 change: 1 addition & 0 deletions client.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ func (c *Client) SetRetries() *Client {
addRetryConditional(tooManyRequestsRetryCondition).
addRetryConditional(serviceUnavailableRetryCondition).
addRetryConditional(requestTimeoutRetryCondition).
addRetryConditional(requestGOAWAYRetryCondition).
addRetryConditional(requestNGINXRetryCondition).
SetRetryMaxWaitTime(APIRetryMaxWaitTime)
configureRetries(c)
Expand Down
5 changes: 4 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ require (
gopkg.in/ini.v1 v1.66.6
)

require golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
require (
golang.org/x/net v0.0.0-20190628185345-da137c7871d7
golang.org/x/text v0.3.0 // indirect
)

go 1.18

Expand Down
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
6 changes: 6 additions & 0 deletions retries.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
package linodego

import (
"errors"
"log"
"net/http"
"strconv"
"time"

"github.com/go-resty/resty/v2"
"golang.org/x/net/http2"
)

const (
Expand Down Expand Up @@ -74,6 +76,10 @@ func requestTimeoutRetryCondition(r *resty.Response, _ error) bool {
return r.StatusCode() == http.StatusRequestTimeout
}

func requestGOAWAYRetryCondition(_ *resty.Response, e error) bool {
return errors.As(e, &http2.GoAwayError{})
}

func requestNGINXRetryCondition(r *resty.Response, _ error) bool {
return r.StatusCode() == http.StatusBadRequest &&
r.Header().Get("Server") == "nginx" &&
Expand Down
2 changes: 1 addition & 1 deletion test/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/jarcoal/httpmock v1.2.0
github.com/linode/linodego v0.20.1
github.com/linode/linodego/k8s v0.0.0-00010101000000-000000000000
golang.org/x/net v0.0.0-20211209124913-491a49abca63
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
k8s.io/client-go v0.23.4
)
Expand All @@ -23,7 +24,6 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/text v0.3.7 // indirect
Expand Down
27 changes: 27 additions & 0 deletions test/integration/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package integration

import (
"context"
"golang.org/x/net/http2"
"net/http"
"testing"

Expand Down Expand Up @@ -44,3 +45,29 @@ func TestClient_NGINXRetry(t *testing.T) {
t.Fatalf("retry checks did not finish")
}
}

func TestClient_GoAwayRetry(t *testing.T) {
client := createMockClient(t)

step := 0

httpmock.RegisterRegexpResponder("PUT",
mockRequestURL(t, "/profile"), func(request *http.Request) (*http.Response, error) {
if step == 0 {
step++
return nil, http2.GoAwayError{}
}

step++
return httpmock.NewJsonResponse(200, nil)
})

if _, err := client.UpdateProfile(context.Background(),
linodego.ProfileUpdateOptions{}); err != nil {
t.Fatal(err)
}

if step != 2 {
t.Fatalf("retry checks did not finish")
}
}
456 changes: 127 additions & 329 deletions test/integration/fixtures/TestInstanceBackups_List.yaml

Large diffs are not rendered by default.

145 changes: 21 additions & 124 deletions test/integration/fixtures/TestInstance_ConfigInterfaces_Update.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,111 +51,7 @@ interactions:
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "ap-south", "country": "sg", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Object Storage", "GPU Linodes", "Kubernetes", "Cloud Firewall",
"Block Storage Migrations", "Managed Databases"], "status": "ok", "resolvers":
{"ipv4": "139.162.11.5,139.162.13.5,139.162.14.5,139.162.15.5,139.162.16.5,139.162.21.5,139.162.27.5,103.3.60.18,103.3.60.19,103.3.60.20",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "eu-central", "country": "de", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Object Storage", "GPU Linodes", "Kubernetes", "Cloud Firewall",
"Vlans", "Block Storage Migrations", "Managed Databases"], "status": "ok", "resolvers":
{"ipv4": "139.162.130.5,139.162.131.5,139.162.132.5,139.162.133.5,139.162.134.5,139.162.135.5,139.162.136.5,139.162.137.5,139.162.138.5,139.162.139.5",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "ap-northeast", "country": "jp", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Kubernetes", "Cloud Firewall", "Block Storage Migrations",
"Managed Databases"], "status": "ok", "resolvers": {"ipv4": "139.162.66.5,139.162.67.5,139.162.68.5,139.162.69.5,139.162.70.5,139.162.71.5,139.162.72.5,139.162.73.5,139.162.74.5,139.162.75.5",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}}],
"page": 1, "pages": 1, "results": 11}'
headers:
Access-Control-Allow-Credentials:
- "true"
Access-Control-Allow-Headers:
- Authorization, Origin, X-Requested-With, Content-Type, Accept, X-Filter
Access-Control-Allow-Methods:
- HEAD, GET, OPTIONS, POST, PUT, DELETE
Access-Control-Allow-Origin:
- '*'
Access-Control-Expose-Headers:
- X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Status
Cache-Control:
- private, max-age=900
- private, max-age=60, s-maxage=60
Content-Security-Policy:
- default-src 'none'
Content-Type:
- application/json
Server:
- nginx
Strict-Transport-Security:
- max-age=31536000
Vary:
- Authorization, X-Filter
- Authorization, X-Filter
X-Accepted-Oauth-Scopes:
- '*'
X-Content-Type-Options:
- nosniff
X-Frame-Options:
- DENY
- DENY
X-Oauth-Scopes:
- '*'
X-Ratelimit-Limit:
- "800"
X-Xss-Protection:
- 1; mode=block
status: 200 OK
code: 200
duration: ""
- request:
body: ""
form: {}
headers:
Accept:
- application/json
Content-Type:
- application/json
User-Agent:
- linodego/dev https://github.com/linode/linodego
url: https://api.linode.com/v4beta/regions
method: GET
response:
body: '{"data": [{"id": "ap-west", "country": "in", "capabilities": ["Linodes",
"NodeBalancers", "Block Storage", "GPU Linodes", "Kubernetes", "Cloud Firewall",
"Vlans", "Block Storage Migrations", "Managed Databases"], "status": "ok", "resolvers":
{"ipv4": "172.105.34.5,172.105.35.5,172.105.36.5,172.105.37.5,172.105.38.5,172.105.39.5,172.105.40.5,172.105.41.5,172.105.42.5,172.105.43.5",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "ca-central", "country": "ca", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Kubernetes", "Cloud Firewall", "Vlans", "Block Storage Migrations",
"Managed Databases"], "status": "ok", "resolvers": {"ipv4": "172.105.0.5,172.105.3.5,172.105.4.5,172.105.5.5,172.105.6.5,172.105.7.5,172.105.8.5,172.105.9.5,172.105.10.5,172.105.11.5",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "ap-southeast", "country": "au", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Kubernetes", "Cloud Firewall", "Vlans", "Block Storage Migrations",
"Managed Databases"], "status": "ok", "resolvers": {"ipv4": "172.105.166.5,172.105.169.5,172.105.168.5,172.105.172.5,172.105.162.5,172.105.170.5,172.105.167.5,172.105.171.5,172.105.181.5,172.105.161.5",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "us-central", "country": "us", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Kubernetes", "Cloud Firewall", "Block Storage Migrations",
"Managed Databases"], "status": "ok", "resolvers": {"ipv4": "72.14.179.5,72.14.188.5,173.255.199.5,66.228.53.5,96.126.122.5,96.126.124.5,96.126.127.5,198.58.107.5,198.58.111.5,23.239.24.5",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "us-west", "country": "us", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Kubernetes", "Cloud Firewall", "Block Storage Migrations",
"Managed Databases"], "status": "ok", "resolvers": {"ipv4": "173.230.145.5,173.230.147.5,173.230.155.5,173.255.212.5,173.255.219.5,173.255.241.5,173.255.243.5,173.255.244.5,74.207.241.5,74.207.242.5",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "us-southeast", "country": "us", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Object Storage", "GPU Linodes", "Kubernetes", "Cloud Firewall",
"Vlans", "Block Storage Migrations", "Managed Databases"], "status": "ok", "resolvers":
{"ipv4": "74.207.231.5,173.230.128.5,173.230.129.5,173.230.136.5,173.230.140.5,66.228.59.5,66.228.62.5,50.116.35.5,50.116.41.5,23.239.18.5",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "us-east", "country": "us", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Object Storage", "GPU Linodes", "Kubernetes", "Cloud Firewall",
"Bare Metal", "Block Storage Migrations", "Managed Databases"], "status": "ok",
"resolvers": {"ipv4": "66.228.42.5,96.126.106.5,50.116.53.5,50.116.58.5,50.116.61.5,50.116.62.5,66.175.211.5,97.107.133.4,207.192.69.4,207.192.69.5",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "eu-west", "country": "uk", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Kubernetes", "Cloud Firewall", "Vlans", "Block Storage Migrations",
"Managed Databases"], "status": "ok", "resolvers": {"ipv4": "178.79.182.5,176.58.107.5,176.58.116.5,176.58.121.5,151.236.220.5,212.71.252.5,212.71.253.5,109.74.192.20,109.74.193.20,109.74.194.20",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "ap-south", "country": "sg", "capabilities": ["Linodes", "NodeBalancers",
"Block Storage", "Object Storage", "GPU Linodes", "Kubernetes", "Cloud Firewall",
"Block Storage Migrations", "Managed Databases"], "status": "ok", "resolvers":
{"ipv4": "139.162.11.5,139.162.13.5,139.162.14.5,139.162.15.5,139.162.16.5,139.162.21.5,139.162.27.5,103.3.60.18,103.3.60.19,103.3.60.20",
"ipv6": "1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678,1234::5678"}},
{"id": "eu-central", "country": "de", "capabilities": ["Linodes", "NodeBalancers",
Expand Down Expand Up @@ -210,7 +106,7 @@ interactions:
code: 200
duration: ""
- request:
body: '{"region":"ap-west","type":"g6-nanode-1","label":"go-test-ins-wo-disk-z871lszz170x","booted":false}'
body: '{"region":"ap-west","type":"g6-nanode-1","label":"go-test-ins-wo-disk-k20sl49e30fs","booted":false}'
form: {}
headers:
Accept:
Expand All @@ -222,14 +118,15 @@ interactions:
url: https://api.linode.com/v4beta/linode/instances
method: POST
response:
body: '{"id": 39423194, "label": "go-test-ins-wo-disk-z871lszz170x", "group":
body: '{"id": 41004307, "label": "go-test-ins-wo-disk-k20sl49e30fs", "group":
"", "status": "provisioning", "created": "2018-01-02T03:04:05", "updated": "2018-01-02T03:04:05",
"type": "g6-nanode-1", "ipv4": ["139.144.0.144"], "ipv6": "1234::5678/128",
"type": "g6-nanode-1", "ipv4": ["172.105.59.146"], "ipv6": "1234::5678/128",
"image": null, "region": "ap-west", "specs": {"disk": 25600, "memory": 1024,
"vcpus": 1, "gpus": 0, "transfer": 1000}, "alerts": {"cpu": 90, "network_in":
10, "network_out": 10, "transfer_quota": 80, "io": 10000}, "backups": {"enabled":
false, "schedule": {"day": null, "window": null}, "last_successful": null},
"hypervisor": "kvm", "watchdog_enabled": true, "tags": []}'
false, "available": false, "schedule": {"day": null, "window": null}, "last_successful":
null}, "hypervisor": "kvm", "watchdog_enabled": true, "tags": [], "host_uuid":
"0346babe1bc45d9259a93fb78b6b7858acaf41c4"}'
headers:
Access-Control-Allow-Credentials:
- "true"
Expand All @@ -244,7 +141,7 @@ interactions:
Cache-Control:
- private, max-age=60, s-maxage=60
Content-Length:
- "636"
- "714"
Content-Security-Policy:
- default-src 'none'
Content-Type:
Expand Down Expand Up @@ -272,7 +169,7 @@ interactions:
code: 200
duration: ""
- request:
body: '{"label":"go-test-conf-g556g8v3la9s","devices":{},"interfaces":null}'
body: '{"label":"go-test-conf-1781gp3du8mu","devices":{},"interfaces":null}'
form: {}
headers:
Accept:
Expand All @@ -281,10 +178,10 @@ interactions:
- application/json
User-Agent:
- linodego/dev https://github.com/linode/linodego
url: https://api.linode.com/v4beta/linode/instances/39423194/configs
url: https://api.linode.com/v4beta/linode/instances/41004307/configs
method: POST
response:
body: '{"id": 41968603, "label": "go-test-conf-g556g8v3la9s", "helpers": {"updatedb_disabled":
body: '{"id": 43613864, "label": "go-test-conf-1781gp3du8mu", "helpers": {"updatedb_disabled":
true, "distro": true, "modules_dep": true, "network": true, "devtmpfs_automount":
true}, "kernel": "linode/latest-64bit", "comments": "", "memory_limit": 0, "created":
"2018-01-02T03:04:05", "updated": "2018-01-02T03:04:05", "root_device": "/dev/sda",
Expand Down Expand Up @@ -333,7 +230,7 @@ interactions:
code: 200
duration: ""
- request:
body: '{"comments":"","interfaces":[{"ipam_address":"","label":"","purpose":"public"},{"ipam_address":"","label":"go-test-ins-wo-disk-z871lszz170x-r","purpose":"vlan"}],"memory_limit":0,"init_rd":null}'
body: '{"comments":"","interfaces":[{"ipam_address":"","label":"","purpose":"public"},{"ipam_address":"","label":"go-test-ins-wo-disk-k20sl49e30fs-r","purpose":"vlan"}],"memory_limit":0,"init_rd":null}'
form: {}
headers:
Accept:
Expand All @@ -342,17 +239,17 @@ interactions:
- application/json
User-Agent:
- linodego/dev https://github.com/linode/linodego
url: https://api.linode.com/v4beta/linode/instances/39423194/configs/41968603
url: https://api.linode.com/v4beta/linode/instances/41004307/configs/43613864
method: PUT
response:
body: '{"id": 41968603, "label": "go-test-conf-g556g8v3la9s", "helpers": {"updatedb_disabled":
body: '{"id": 43613864, "label": "go-test-conf-1781gp3du8mu", "helpers": {"updatedb_disabled":
true, "distro": true, "modules_dep": true, "network": true, "devtmpfs_automount":
true}, "kernel": "linode/latest-64bit", "comments": "", "memory_limit": 0, "created":
"2018-01-02T03:04:05", "updated": "2018-01-02T03:04:05", "root_device": "/dev/sda",
"devices": {"sda": null, "sdb": null, "sdc": null, "sdd": null, "sde": null,
"sdf": null, "sdg": null, "sdh": null}, "initrd": null, "run_level": "default",
"virt_mode": "paravirt", "interfaces": [{"purpose": "public", "ipam_address":
null, "label": null}, {"purpose": "vlan", "ipam_address": "", "label": "go-test-ins-wo-disk-z871lszz170x-r"}]}'
null, "label": null}, {"purpose": "vlan", "ipam_address": "", "label": "go-test-ins-wo-disk-k20sl49e30fs-r"}]}'
headers:
Access-Control-Allow-Credentials:
- "true"
Expand Down Expand Up @@ -404,17 +301,17 @@ interactions:
- application/json
User-Agent:
- linodego/dev https://github.com/linode/linodego
url: https://api.linode.com/v4beta/linode/instances/39423194/configs/41968603
url: https://api.linode.com/v4beta/linode/instances/41004307/configs/43613864
method: GET
response:
body: '{"id": 41968603, "label": "go-test-conf-g556g8v3la9s", "helpers": {"updatedb_disabled":
body: '{"id": 43613864, "label": "go-test-conf-1781gp3du8mu", "helpers": {"updatedb_disabled":
true, "distro": true, "modules_dep": true, "network": true, "devtmpfs_automount":
true}, "kernel": "linode/latest-64bit", "comments": "", "memory_limit": 0, "created":
"2018-01-02T03:04:05", "updated": "2018-01-02T03:04:05", "root_device": "/dev/sda",
"devices": {"sda": null, "sdb": null, "sdc": null, "sdd": null, "sde": null,
"sdf": null, "sdg": null, "sdh": null}, "initrd": null, "run_level": "default",
"virt_mode": "paravirt", "interfaces": [{"purpose": "public", "ipam_address":
null, "label": null}, {"purpose": "vlan", "ipam_address": "", "label": "go-test-ins-wo-disk-z871lszz170x-r"}]}'
null, "label": null}, {"purpose": "vlan", "ipam_address": "", "label": "go-test-ins-wo-disk-k20sl49e30fs-r"}]}'
headers:
Access-Control-Allow-Credentials:
- "true"
Expand Down Expand Up @@ -468,17 +365,17 @@ interactions:
- application/json
User-Agent:
- linodego/dev https://github.com/linode/linodego
url: https://api.linode.com/v4beta/linode/instances/39423194/configs/41968603
url: https://api.linode.com/v4beta/linode/instances/41004307/configs/43613864
method: PUT
response:
body: '{"id": 41968603, "label": "go-test-conf-g556g8v3la9s", "helpers": {"updatedb_disabled":
body: '{"id": 43613864, "label": "go-test-conf-1781gp3du8mu", "helpers": {"updatedb_disabled":
true, "distro": true, "modules_dep": true, "network": true, "devtmpfs_automount":
true}, "kernel": "linode/latest-64bit", "comments": "", "memory_limit": 0, "created":
"2018-01-02T03:04:05", "updated": "2018-01-02T03:04:05", "root_device": "/dev/sda",
"devices": {"sda": null, "sdb": null, "sdc": null, "sdd": null, "sde": null,
"sdf": null, "sdg": null, "sdh": null}, "initrd": null, "run_level": "default",
"virt_mode": "paravirt", "interfaces": [{"purpose": "public", "ipam_address":
null, "label": null}, {"purpose": "vlan", "ipam_address": "", "label": "go-test-ins-wo-disk-z871lszz170x-r"}]}'
null, "label": null}, {"purpose": "vlan", "ipam_address": "", "label": "go-test-ins-wo-disk-k20sl49e30fs-r"}]}'
headers:
Access-Control-Allow-Credentials:
- "true"
Expand Down Expand Up @@ -530,7 +427,7 @@ interactions:
- application/json
User-Agent:
- linodego/dev https://github.com/linode/linodego
url: https://api.linode.com/v4beta/linode/instances/39423194
url: https://api.linode.com/v4beta/linode/instances/41004307
method: DELETE
response:
body: '{}'
Expand Down
Loading

0 comments on commit 5451945

Please sign in to comment.