topology recovery: supporting {instanceType}, {isMaster}, {isCoMaster} placeholders

docs/configuration-recovery.md

@@ -104,6 +104,9 @@ This information is passed independently in two ways, and you may choose to use
 1. Environment variables: `orchestrator` will set the following, which can be retrieved by your hooks:
 
 - `ORC_FAILURE_TYPE`
+- `ORC_INSTANCE_TYPE` ("master", "co-master", "intermediate-master")
+- `ORC_IS_MASTER` (true/false)
+- `ORC_IS_CO_MASTER` (true/false)
 - `ORC_FAILURE_DESCRIPTION`
 - `ORC_FAILED_HOST`
 - `ORC_FAILED_PORT`
@@ -129,6 +132,9 @@ And, in the event a recovery was successful:
 2. Command line text replacement. `orchestrator` replaces the following magic tokens in your `*Proccesses` commands:
 
 - `{failureType}`
+- `{instanceType}` ("master", "co-master", "intermediate-master")
+- `{isMaster}` (true/false)
+- `{isCoMaster}` (true/false)
 - `{failureDescription}`
 - `{failedHost}`
 - `{failedPort}`

go/config/config.go

@@ -226,11 +226,11 @@ type Configuration struct {
 	RecoverMasterClusterFilters                []string          // Only do master recovery on clusters matching these regexp patterns (of course the ".*" pattern matches everything)
 	RecoverIntermediateMasterClusterFilters    []string          // Only do IM recovery on clusters matching these regexp patterns (of course the ".*" pattern matches everything)
 	ProcessesShellCommand                      string            // Shell that executes command scripts
-	OnFailureDetectionProcesses                []string          // Processes to execute when detecting a failover scenario (before making a decision whether to failover or not). May and should use some of these placeholders: {failureType}, {failureDescription}, {command}, {failedHost}, {failureCluster}, {failureClusterAlias}, {failureClusterDomain}, {failedPort}, {successorHost}, {successorPort}, {successorAlias}, {countReplicas}, {replicaHosts}, {isDowntimed}, {autoMasterRecovery}, {autoIntermediateMasterRecovery}
-	PreGracefulTakeoverProcesses               []string          // Processes to execute before doing a failover (aborting operation should any once of them exits with non-zero code; order of execution undefined). May and should use some of these placeholders: {failureType}, {failureDescription}, {command}, {failedHost}, {failureCluster}, {failureClusterAlias}, {failureClusterDomain}, {failedPort}, {successorHost}, {successorPort}, {countReplicas}, {replicaHosts}, {isDowntimed}
-	PreFailoverProcesses                       []string          // Processes to execute before doing a failover (aborting operation should any once of them exits with non-zero code; order of execution undefined). May and should use some of these placeholders: {failureType}, {failureDescription}, {command}, {failedHost}, {failureCluster}, {failureClusterAlias}, {failureClusterDomain}, {failedPort}, {countReplicas}, {replicaHosts}, {isDowntimed}
-	PostFailoverProcesses                      []string          // Processes to execute after doing a failover (order of execution undefined). May and should use some of these placeholders: {failureType}, {failureDescription}, {command}, {failedHost}, {failureCluster}, {failureClusterAlias}, {failureClusterDomain}, {failedPort}, {successorHost}, {successorPort}, {successorAlias}, {countReplicas}, {replicaHosts}, {isDowntimed}, {isSuccessful}, {lostReplicas}, {countLostReplicas}
-	PostUnsuccessfulFailoverProcesses          []string          // Processes to execute after a not-completely-successful failover (order of execution undefined). May and should use some of these placeholders: {failureType}, {failureDescription}, {command}, {failedHost}, {failureCluster}, {failureClusterAlias}, {failureClusterDomain}, {failedPort}, {successorHost}, {successorPort}, {successorAlias}, {countReplicas}, {replicaHosts}, {isDowntimed}, {isSuccessful}, {lostReplicas}, {countLostReplicas}
+	OnFailureDetectionProcesses                []string          // Processes to execute when detecting a failover scenario (before making a decision whether to failover or not). May and should use some of these placeholders: {failureType}, {instanceType}, {isMaster}, {isCoMaster}, {failureDescription}, {command}, {failedHost}, {failureCluster}, {failureClusterAlias}, {failureClusterDomain}, {failedPort}, {successorHost}, {successorPort}, {successorAlias}, {countReplicas}, {replicaHosts}, {isDowntimed}, {autoMasterRecovery}, {autoIntermediateMasterRecovery}
+	PreGracefulTakeoverProcesses               []string          // Processes to execute before doing a failover (aborting operation should any once of them exits with non-zero code; order of execution undefined). May and should use some of these placeholders: {failureType}, {instanceType}, {isMaster}, {isCoMaster}, {failureDescription}, {command}, {failedHost}, {failureCluster}, {failureClusterAlias}, {failureClusterDomain}, {failedPort}, {successorHost}, {successorPort}, {countReplicas}, {replicaHosts}, {isDowntimed}
+	PreFailoverProcesses                       []string          // Processes to execute before doing a failover (aborting operation should any once of them exits with non-zero code; order of execution undefined). May and should use some of these placeholders: {failureType}, {instanceType}, {isMaster}, {isCoMaster}, {failureDescription}, {command}, {failedHost}, {failureCluster}, {failureClusterAlias}, {failureClusterDomain}, {failedPort}, {countReplicas}, {replicaHosts}, {isDowntimed}
+	PostFailoverProcesses                      []string          // Processes to execute after doing a failover (order of execution undefined). May and should use some of these placeholders: {failureType}, {instanceType}, {isMaster}, {isCoMaster}, {failureDescription}, {command}, {failedHost}, {failureCluster}, {failureClusterAlias}, {failureClusterDomain}, {failedPort}, {successorHost}, {successorPort}, {successorAlias}, {countReplicas}, {replicaHosts}, {isDowntimed}, {isSuccessful}, {lostReplicas}, {countLostReplicas}
+	PostUnsuccessfulFailoverProcesses          []string          // Processes to execute after a not-completely-successful failover (order of execution undefined). May and should use some of these placeholders: {failureType}, {instanceType}, {isMaster}, {isCoMaster}, {failureDescription}, {command}, {failedHost}, {failureCluster}, {failureClusterAlias}, {failureClusterDomain}, {failedPort}, {successorHost}, {successorPort}, {successorAlias}, {countReplicas}, {replicaHosts}, {isDowntimed}, {isSuccessful}, {lostReplicas}, {countLostReplicas}
 	PostMasterFailoverProcesses                []string          // Processes to execute after doing a master failover (order of execution undefined). Uses same placeholders as PostFailoverProcesses
 	PostIntermediateMasterFailoverProcesses    []string          // Processes to execute after doing a master failover (order of execution undefined). Uses same placeholders as PostFailoverProcesses
 	PostGracefulTakeoverProcesses              []string          // Processes to execute after runnign a graceful master takeover. Uses same placeholders as PostFailoverProcesses

go/inst/analysis.go

@@ -100,6 +100,14 @@ const (
 	GracefulMasterTakeoverCommandHint string = "graceful-master-takeover"
 )
 
+type AnalysisInstanceType string
+
+const (
+	AnalysisInstanceTypeMaster             AnalysisInstanceType = "master"
+	AnalysisInstanceTypeCoMaster           AnalysisInstanceType = "co-master"
+	AnalysisInstanceTypeIntermediateMaster AnalysisInstanceType = "intermediate-master"
+)
+
 // ReplicationAnalysis notes analysis on replication chain status, per instance
 type ReplicationAnalysis struct {
 	AnalyzedInstanceKey                       InstanceKey
@@ -178,6 +186,17 @@ func (this *ReplicationAnalysis) AnalysisString() string {
 	return strings.Join(result, ", ")
 }
 
+// Get a string description of the analyzed instance type (master? co-master? intermediate-master?)
+func (this *ReplicationAnalysis) GetAnalysisInstanceType() AnalysisInstanceType {
+	if this.IsCoMaster {
+		return AnalysisInstanceTypeCoMaster
+	}
+	if this.IsMaster {
+		return AnalysisInstanceTypeMaster
+	}
+	return AnalysisInstanceTypeIntermediateMaster
+}
+
 // ValidSecondsFromSeenToLastAttemptedCheck returns the maximum allowed elapsed time
 // between last_attempted_check to last_checked before we consider the instance as invalid.
 func ValidSecondsFromSeenToLastAttemptedCheck() uint {

go/inst/analysis_test.go

@@ -17,10 +17,11 @@
 package inst
 
 import (
+	"testing"
+
 	"github.com/github/orchestrator/go/config"
 	"github.com/openark/golib/log"
-	// test "github.com/openark/golib/tests"
-	"testing"
+	test "github.com/openark/golib/tests"
 )
 
 func init() {
@@ -30,20 +31,20 @@ func init() {
 }
 
 func TestGetAnalysisInstanceType(t *testing.T) {
-	// {
-	// 	analysis := &ReplicationAnalysis{}
-	// 	test.S(t).ExpectEquals(string(analysis.GetAnalysisInstanceType()), "intermediate-master")
-	// }
-	// {
-	// 	analysis := &ReplicationAnalysis{IsMaster: true}
-	// 	test.S(t).ExpectEquals(string(analysis.GetAnalysisInstanceType()), "master")
-	// }
-	// {
-	// 	analysis := &ReplicationAnalysis{IsCoMaster: true}
-	// 	test.S(t).ExpectEquals(string(analysis.GetAnalysisInstanceType()), "co-master")
-	// }
-	// {
-	// 	analysis := &ReplicationAnalysis{IsMaster: true, IsCoMaster: true}
-	// 	test.S(t).ExpectEquals(string(analysis.GetAnalysisInstanceType()), "co-master")
-	// }
+	{
+		analysis := &ReplicationAnalysis{}
+		test.S(t).ExpectEquals(string(analysis.GetAnalysisInstanceType()), "intermediate-master")
+	}
+	{
+		analysis := &ReplicationAnalysis{IsMaster: true}
+		test.S(t).ExpectEquals(string(analysis.GetAnalysisInstanceType()), "master")
+	}
+	{
+		analysis := &ReplicationAnalysis{IsCoMaster: true}
+		test.S(t).ExpectEquals(string(analysis.GetAnalysisInstanceType()), "co-master")
+	}
+	{
+		analysis := &ReplicationAnalysis{IsMaster: true, IsCoMaster: true}
+		test.S(t).ExpectEquals(string(analysis.GetAnalysisInstanceType()), "co-master")
+	}
 }

go/logic/topology_recovery.go

@@ -261,6 +261,9 @@ func prepareCommand(command string, topologyRecovery *TopologyRecovery) (result
 		async = true
 	}
 	command = strings.Replace(command, "{failureType}", string(analysisEntry.Analysis), -1)
+	command = strings.Replace(command, "{instanceType}", string(analysisEntry.GetAnalysisInstanceType()), -1)
+	command = strings.Replace(command, "{isMaster}", fmt.Sprintf("%t", analysisEntry.IsMaster), -1)
+	command = strings.Replace(command, "{isCoMaster}", fmt.Sprintf("%t", analysisEntry.IsCoMaster), -1)
 	command = strings.Replace(command, "{failureDescription}", analysisEntry.Description, -1)
 	command = strings.Replace(command, "{command}", analysisEntry.CommandHint, -1)
 	command = strings.Replace(command, "{failedHost}", analysisEntry.AnalyzedInstanceKey.Hostname, -1)
@@ -299,6 +302,9 @@ func applyEnvironmentVariables(topologyRecovery *TopologyRecovery) []string {
 	analysisEntry := &topologyRecovery.AnalysisEntry
 	env := goos.Environ()
 	env = append(env, fmt.Sprintf("ORC_FAILURE_TYPE=%s", string(analysisEntry.Analysis)))
+	env = append(env, fmt.Sprintf("ORC_INSTANCE_TYPE=%s", string(analysisEntry.GetAnalysisInstanceType())))
+	env = append(env, fmt.Sprintf("ORC_IS_MASTER=%t", analysisEntry.IsMaster))
+	env = append(env, fmt.Sprintf("ORC_IS_CO_MASTER=%t", analysisEntry.IsCoMaster))
 	env = append(env, fmt.Sprintf("ORC_FAILURE_DESCRIPTION=%s", analysisEntry.Description))
 	env = append(env, fmt.Sprintf("ORC_COMMAND=%s", analysisEntry.CommandHint))
 	env = append(env, fmt.Sprintf("ORC_FAILED_HOST=%s", analysisEntry.AnalyzedInstanceKey.Hostname))