Improve retries

ydb-platform · Sep 6, 2024 · 10ae9b5 · 10ae9b5
1 parent e5750ca
commit 10ae9b5
Show file tree

Hide file tree

Showing 9 changed files with 24 additions and 119 deletions.
diff --git a/app/server/datasource/rdbms/clickhouse/retry.go b/app/server/datasource/rdbms/clickhouse/retry.go
diff --git a/app/server/datasource/rdbms/data_source_factory.go b/app/server/datasource/rdbms/data_source_factory.go
@@ -86,7 +86,7 @@ func NewDataSourceFactory(
 			TypeMapper:        clickhouseTypeMapper,
 			SchemaProvider:    rdbms_utils.NewDefaultSchemaProvider(clickhouseTypeMapper, clickhouse.TableMetadataQuery),
 			RetrierSet: &retry.RetrierSet{
-				MakeConnection: retry.NewRetrierFromConfig(cfg.Clickhouse.ExponentialBackoff, clickhouse.ErrorCheckerMakeConnection),
+				MakeConnection: retry.NewRetrierFromConfig(cfg.Clickhouse.ExponentialBackoff, retry.ErrorCheckerMakeConnectionCommon),
 				Query:          retry.NewRetrierFromConfig(cfg.Clickhouse.ExponentialBackoff, retry.ErrorCheckerNoop),
 			},
 		},
@@ -103,7 +103,7 @@ func NewDataSourceFactory(
 						schemaGetters[api_common.EDataSourceKind_POSTGRESQL](request.DataSourceInstance))
 				}),
 			RetrierSet: &retry.RetrierSet{
-				MakeConnection: retry.NewRetrierFromConfig(cfg.Postgresql.ExponentialBackoff, postgresql.ErrorCheckerMakeConnection),
+				MakeConnection: retry.NewRetrierFromConfig(cfg.Postgresql.ExponentialBackoff, retry.ErrorCheckerMakeConnectionCommon),
 				Query:          retry.NewRetrierFromConfig(cfg.Postgresql.ExponentialBackoff, retry.ErrorCheckerNoop),
 			},
 		},
@@ -113,7 +113,7 @@ func NewDataSourceFactory(
 			TypeMapper:        ydbTypeMapper,
 			SchemaProvider:    ydb.NewSchemaProvider(ydbTypeMapper),
 			RetrierSet: &retry.RetrierSet{
-				MakeConnection: retry.NewRetrierFromConfig(cfg.Ydb.ExponentialBackoff, ydb.ErrorCheckerMakeConnection),
+				MakeConnection: retry.NewRetrierFromConfig(cfg.Ydb.ExponentialBackoff, retry.ErrorCheckerMakeConnectionCommon),
 				Query:          retry.NewRetrierFromConfig(cfg.Ydb.ExponentialBackoff, ydb.ErrorCheckerQuery),
 			},
 		},
@@ -123,7 +123,7 @@ func NewDataSourceFactory(
 			TypeMapper:        msSQLServerTypeMapper,
 			SchemaProvider:    rdbms_utils.NewDefaultSchemaProvider(msSQLServerTypeMapper, ms_sql_server.TableMetadataQuery),
 			RetrierSet: &retry.RetrierSet{
-				MakeConnection: retry.NewRetrierFromConfig(cfg.MsSqlServer.ExponentialBackoff, ms_sql_server.ErrorCheckerMakeConnection),
+				MakeConnection: retry.NewRetrierFromConfig(cfg.MsSqlServer.ExponentialBackoff, retry.ErrorCheckerMakeConnectionCommon),
 				Query:          retry.NewRetrierFromConfig(cfg.MsSqlServer.ExponentialBackoff, retry.ErrorCheckerNoop),
 			},
 		},
@@ -133,7 +133,7 @@ func NewDataSourceFactory(
 			TypeMapper:        mysqlTypeMapper,
 			SchemaProvider:    rdbms_utils.NewDefaultSchemaProvider(mysqlTypeMapper, mysql.TableMetadataQuery),
 			RetrierSet: &retry.RetrierSet{
-				MakeConnection: retry.NewRetrierFromConfig(cfg.Mysql.ExponentialBackoff, mysql.ErrorCheckerMakeConnection),
+				MakeConnection: retry.NewRetrierFromConfig(cfg.Mysql.ExponentialBackoff, retry.ErrorCheckerMakeConnectionCommon),
 				Query:          retry.NewRetrierFromConfig(cfg.Mysql.ExponentialBackoff, retry.ErrorCheckerNoop),
 			},
 		},
@@ -150,7 +150,7 @@ func NewDataSourceFactory(
 						schemaGetters[api_common.EDataSourceKind_GREENPLUM](request.DataSourceInstance))
 				}),
 			RetrierSet: &retry.RetrierSet{
-				MakeConnection: retry.NewRetrierFromConfig(cfg.Greenplum.ExponentialBackoff, postgresql.ErrorCheckerMakeConnection),
+				MakeConnection: retry.NewRetrierFromConfig(cfg.Greenplum.ExponentialBackoff, retry.ErrorCheckerMakeConnectionCommon),
 				Query:          retry.NewRetrierFromConfig(cfg.Greenplum.ExponentialBackoff, retry.ErrorCheckerNoop),
 			},
 		},

diff --git a/app/server/datasource/rdbms/ms_sql_server/retry.go b/app/server/datasource/rdbms/ms_sql_server/retry.go
diff --git a/app/server/datasource/rdbms/mysql/retry.go b/app/server/datasource/rdbms/mysql/retry.go
diff --git a/app/server/datasource/rdbms/oracle/retry.go b/app/server/datasource/rdbms/oracle/retry.go
@@ -1,7 +1,6 @@
 package oracle
 
 import (
-	"context"
 	"errors"
 	"net"
 	"strings"
@@ -18,10 +17,5 @@ func ErrorCheckerMakeConnection(err error) bool {
 		}
 	}
 
-	// Also happens on CI
-	if strings.Contains(err.Error(), context.DeadlineExceeded.Error()) {
-		return true
-	}
-
 	return retry.ErrorCheckerMakeConnectionCommon(err)
 }
diff --git a/app/server/datasource/rdbms/postgresql/retry.go b/app/server/datasource/rdbms/postgresql/retry.go
diff --git a/app/server/datasource/rdbms/postgresql/retry_test.go b/app/server/datasource/rdbms/postgresql/retry_test.go
diff --git a/app/server/datasource/rdbms/ydb/retry.go b/app/server/datasource/rdbms/ydb/retry.go
@@ -1,22 +1,10 @@
 package ydb
 
 import (
-	"strings"
-
 	"github.com/ydb-platform/ydb-go-sdk/v3"
 	grpc_codes "google.golang.org/grpc/codes"
-
-	"github.com/ydb-platform/fq-connector-go/app/server/utils/retry"
 )
 
-func ErrorCheckerMakeConnection(err error) bool {
-	if strings.Contains(err.Error(), "server misbehaving") {
-		return true
-	}
-
-	return retry.ErrorCheckerMakeConnectionCommon(err)
-}
-
 func ErrorCheckerQuery(err error) bool {
 	switch {
 	case ydb.IsTransportError(err, grpc_codes.ResourceExhausted):

diff --git a/app/server/utils/retry/error_checker.go b/app/server/utils/retry/error_checker.go
@@ -1,25 +1,39 @@
 package retry
 
 import (
+	"context"
 	"errors"
 	"net"
 	"os"
+	"strings"
 	"syscall"
 )
 
 type ErrorChecker func(err error) bool
 
 func ErrorCheckerMakeConnectionCommon(err error) bool {
 	// 'i/o timeout'
+	if strings.Contains(err.Error(), "i/o timeout") {
+		return true
+	}
+
 	if errors.Is(err, os.ErrDeadlineExceeded) {
 		return true
 	}
 
+	if strings.Contains(err.Error(), context.DeadlineExceeded.Error()) {
+		return true
+	}
+
 	// 'connection refused'
 	if errors.Is(err, syscall.ECONNREFUSED) {
 		return true
 	}
 
+	if strings.Contains(err.Error(), "connection refused") {
+		return true
+	}
+
 	// DNS errors typically caused by CI overload, like 'server misbehaving' and so on.
 	var dnsError *net.DNSError
 	if errors.As(err, &dnsError) {
@@ -28,6 +42,10 @@ func ErrorCheckerMakeConnectionCommon(err error) bool {
 		}
 	}
 
+	if strings.Contains(err.Error(), "server misbehaving") {
+		return true
+	}
+
 	return false
 }