diff --git a/build.sh b/build.sh index 9906159c..2f3c870d 100755 --- a/build.sh +++ b/build.sh @@ -6,7 +6,7 @@ # set -e -RELEASE_VERSION="1.4.590" +RELEASE_VERSION="1.5.0" TOPDIR=/tmp/orchestrator-release export RELEASE_VERSION TOPDIR export GO15VENDOREXPERIMENT=1 diff --git a/go/config/config.go b/go/config/config.go index d018b742..00c51819 100644 --- a/go/config/config.go +++ b/go/config/config.go @@ -108,6 +108,7 @@ type Configuration struct { DetectPhysicalEnvironmentQuery string // Optional query (executed on topology instance) that returns the physical environment of an instance. If provided, must return one row, one column. Overrides PhysicalEnvironmentPattern and useful for installments where env cannot be inferred by hostname DetectSemiSyncEnforcedQuery string // Optional query (executed on topology instance) to determine whether semi-sync is fully enforced for master writes (async fallback is not allowed under any circumstance). If provided, must return one row, one column, value 0 or 1. SupportFuzzyPoolHostnames bool // Should "submit-pool-instances" command be able to pass list of fuzzy instances (fuzzy means non-fqdn, but unique enough to recognize). Defaults 'true', implies more queries on backend db + InstancePoolExpiryMinutes uint // Time after which entries in database_instance_pool are expired (resubmit via `submit-pool-instances`) PromotionIgnoreHostnameFilters []string // Orchestrator will not promote slaves with hostname matching pattern (via -c recovery; for example, avoid promoting dev-dedicated machines) ServeAgentsHttp bool // Spawn another HTTP interface dedicated for orchestrator-agent AgentsUseSSL bool // When "true" orchestrator will listen on agents port with SSL as well as connect to agents via SSL @@ -248,6 +249,7 @@ func newConfiguration() *Configuration { DetectPhysicalEnvironmentQuery: "", DetectSemiSyncEnforcedQuery: "", SupportFuzzyPoolHostnames: true, + InstancePoolExpiryMinutes: 60, PromotionIgnoreHostnameFilters: []string{}, ServeAgentsHttp: false, AgentsUseSSL: false, diff --git a/go/db/db.go b/go/db/db.go index af1d003c..35353477 100644 --- a/go/db/db.go +++ b/go/db/db.go @@ -820,6 +820,11 @@ var generateSQLPatches = []string{ topology_recovery ADD COLUMN successor_alias varchar(128) DEFAULT NULL `, + ` + ALTER TABLE + database_instance + MODIFY cluster_name varchar(128) NOT NULL + `, } // Track if a TLS has already been configured for topology diff --git a/go/inst/instance.go b/go/inst/instance.go index 1c13df30..bfa4bd44 100644 --- a/go/inst/instance.go +++ b/go/inst/instance.go @@ -384,7 +384,7 @@ func (this *Instance) StatusString() string { if this.IsSlave() && !(this.Slave_SQL_Running && this.Slave_IO_Running) { return "nonreplicating" } - if this.IsSlave() && this.SecondsBehindMaster.Int64 > int64(config.Config.ReasonableMaintenanceReplicationLagSeconds) { + if this.IsSlave() && !this.HasReasonableMaintenanceReplicationLag() { return "lag" } return "ok" @@ -439,6 +439,9 @@ func (this *Instance) HumanReadableDescription() string { if this.UsingPseudoGTID { tokens = append(tokens, "P-GTID") } + if this.IsDowntimed { + tokens = append(tokens, "downtimed") + } description := fmt.Sprintf("[%s]", strings.Join(tokens, ",")) return description } diff --git a/go/inst/pool_dao.go b/go/inst/pool_dao.go index 241522cc..1fbd4df4 100644 --- a/go/inst/pool_dao.go +++ b/go/inst/pool_dao.go @@ -21,6 +21,7 @@ import ( "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" + "github.com/outbrain/orchestrator/go/config" "github.com/outbrain/orchestrator/go/db" ) @@ -123,3 +124,16 @@ func ReadClusterPoolInstancesMap(clusterName string, pool string) (*PoolInstance return &poolInstancesMap, nil } + +// ExpirePoolInstances cleans up the database_instance_pool table from expired items +func ExpirePoolInstances() error { + _, err := db.ExecOrchestrator(` + delete + from database_instance_pool + where + registered_at < now() - interval ? minute + `, + config.Config.InstancePoolExpiryMinutes, + ) + return log.Errore(err) +} diff --git a/go/logic/orchestrator.go b/go/logic/orchestrator.go index 6c9512e1..70c2dd9c 100644 --- a/go/logic/orchestrator.go +++ b/go/logic/orchestrator.go @@ -249,6 +249,7 @@ func ContinuousDiscovery() { go inst.ExpireClusterDomainName() go inst.ExpireAudit() go inst.ExpireMasterPositionEquivalence() + go inst.ExpirePoolInstances() go inst.FlushNontrivialResolveCacheToDatabase() go process.ExpireNodesHistory() go process.ExpireAccessTokens() diff --git a/go/process/health_dao.go b/go/process/health_dao.go index 4edd607e..c5ce9c5c 100644 --- a/go/process/health_dao.go +++ b/go/process/health_dao.go @@ -19,11 +19,12 @@ package process import ( "database/sql" "fmt" + "time" + "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" "github.com/outbrain/orchestrator/go/config" "github.com/outbrain/orchestrator/go/db" - "time" ) const registrationPollSeconds = 10