Merge pull request #217 from outbrain/merge-downstream-gh

Merge downstream gh

build.sh

@@ -6,7 +6,7 @@
 #
 set -e
 
-RELEASE_VERSION="1.4.590"
+RELEASE_VERSION="1.5.0"
 TOPDIR=/tmp/orchestrator-release
 export RELEASE_VERSION TOPDIR
 export GO15VENDOREXPERIMENT=1

go/config/config.go

@@ -108,6 +108,7 @@ type Configuration struct {
 	DetectPhysicalEnvironmentQuery               string            // Optional query (executed on topology instance) that returns the physical environment of an instance. If provided, must return one row, one column. Overrides PhysicalEnvironmentPattern and useful for installments where env cannot be inferred by hostname
 	DetectSemiSyncEnforcedQuery                  string            // Optional query (executed on topology instance) to determine whether semi-sync is fully enforced for master writes (async fallback is not allowed under any circumstance). If provided, must return one row, one column, value 0 or 1.
 	SupportFuzzyPoolHostnames                    bool              // Should "submit-pool-instances" command be able to pass list of fuzzy instances (fuzzy means non-fqdn, but unique enough to recognize). Defaults 'true', implies more queries on backend db
+	InstancePoolExpiryMinutes                    uint              // Time after which entries in database_instance_pool are expired (resubmit via `submit-pool-instances`)
 	PromotionIgnoreHostnameFilters               []string          // Orchestrator will not promote slaves with hostname matching pattern (via -c recovery; for example, avoid promoting dev-dedicated machines)
 	ServeAgentsHttp                              bool              // Spawn another HTTP interface dedicated for orchestrator-agent
 	AgentsUseSSL                                 bool              // When "true" orchestrator will listen on agents port with SSL as well as connect to agents via SSL
@@ -248,6 +249,7 @@ func newConfiguration() *Configuration {
 		DetectPhysicalEnvironmentQuery:               "",
 		DetectSemiSyncEnforcedQuery:                  "",
 		SupportFuzzyPoolHostnames:                    true,
+		InstancePoolExpiryMinutes:                    60,
 		PromotionIgnoreHostnameFilters:               []string{},
 		ServeAgentsHttp:                              false,
 		AgentsUseSSL:                                 false,

go/db/db.go

@@ -820,6 +820,11 @@ var generateSQLPatches = []string{
 			topology_recovery
 			ADD COLUMN successor_alias varchar(128) DEFAULT NULL
 	`,
+	`
+		ALTER TABLE
+			database_instance
+			MODIFY cluster_name varchar(128) NOT NULL
+	`,
 }
 
 // Track if a TLS has already been configured for topology

go/inst/instance.go

@@ -384,7 +384,7 @@ func (this *Instance) StatusString() string {
 	if this.IsSlave() && !(this.Slave_SQL_Running && this.Slave_IO_Running) {
 		return "nonreplicating"
 	}
-	if this.IsSlave() && this.SecondsBehindMaster.Int64 > int64(config.Config.ReasonableMaintenanceReplicationLagSeconds) {
+	if this.IsSlave() && !this.HasReasonableMaintenanceReplicationLag() {
 		return "lag"
 	}
 	return "ok"
@@ -439,6 +439,9 @@ func (this *Instance) HumanReadableDescription() string {
 	if this.UsingPseudoGTID {
 		tokens = append(tokens, "P-GTID")
 	}
+	if this.IsDowntimed {
+		tokens = append(tokens, "downtimed")
+	}
 	description := fmt.Sprintf("[%s]", strings.Join(tokens, ","))
 	return description
 }

go/inst/pool_dao.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/outbrain/golib/log"
 	"github.com/outbrain/golib/sqlutils"
+	"github.com/outbrain/orchestrator/go/config"
 	"github.com/outbrain/orchestrator/go/db"
 )
 
@@ -123,3 +124,16 @@ func ReadClusterPoolInstancesMap(clusterName string, pool string) (*PoolInstance
 
 	return &poolInstancesMap, nil
 }
+
+// ExpirePoolInstances cleans up the database_instance_pool table from expired items
+func ExpirePoolInstances() error {
+	_, err := db.ExecOrchestrator(`
+			delete
+				from database_instance_pool
+			where
+				registered_at < now() - interval ? minute
+			`,
+		config.Config.InstancePoolExpiryMinutes,
+	)
+	return log.Errore(err)
+}

go/logic/orchestrator.go

@@ -249,6 +249,7 @@ func ContinuousDiscovery() {
 					go inst.ExpireClusterDomainName()
 					go inst.ExpireAudit()
 					go inst.ExpireMasterPositionEquivalence()
+					go inst.ExpirePoolInstances()
 					go inst.FlushNontrivialResolveCacheToDatabase()
 					go process.ExpireNodesHistory()
 					go process.ExpireAccessTokens()

go/process/health_dao.go

@@ -19,11 +19,12 @@ package process
 import (
 	"database/sql"
 	"fmt"
+	"time"
+
 	"github.com/outbrain/golib/log"
 	"github.com/outbrain/golib/sqlutils"
 	"github.com/outbrain/orchestrator/go/config"
 	"github.com/outbrain/orchestrator/go/db"
-	"time"
 )
 
 const registrationPollSeconds = 10