Orchestrator GUI incorrectly shows recovery option for intermediate database in chained replication

go/inst/analysis_dao.go

@@ -608,9 +608,9 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints)
 				a.Analysis = AllIntermediateMasterReplicasFailingToConnectOrDead
 				a.Description = "Intermediate master is reachable but all of its replicas are failing to connect"
 				//
-			} else if !a.IsMaster && a.LastCheckValid && a.CountReplicas > 0 && a.CountValidReplicatingReplicas == 0 {
+			} else if !a.IsMaster && a.LastCheckValid && a.CountReplicas > 0 && a.CountValidReplicas > 0 && a.CountValidReplicatingReplicas == 0 {
 				a.Analysis = AllIntermediateMasterReplicasNotReplicating
-				a.Description = "Intermediate master is reachable but none of its replicas is replicating"
+				a.Description = "Intermediate master is reachable but none of its reachable replicas is replicating"
 				//
 			} else if a.IsBinlogServer && a.IsFailingToConnectToMaster {
 				a.Analysis = BinlogServerFailingToConnectToMaster

go/inst/instance_dao.go

@@ -355,13 +355,21 @@ func ReadTopologyInstanceBufferable(instanceKey *InstanceKey, bufferWrites bool,
 
 	latency.Start("instance")
 	db, err := db.OpenDiscovery(instanceKey.Hostname, instanceKey.Port)
-	latency.Stop("instance")
 	if err != nil {
+		latency.Stop("instance")
 		goto Cleanup
 	}
 
+	// Even if the instance is dead, we need its key below to update
+	// the backend database's timestamps
 	instance.Key = *instanceKey
 
+	err = db.Ping()
+	if err != nil {
+		goto Cleanup
+	}
+	latency.Stop("instance")
+
 	if isMaxScale, resolvedHostname, err = instance.checkMaxScale(db, latency); err != nil {
 		// We do not "goto Cleanup" here, although it should be the correct flow.
 		// Reason is 5.7's new security feature that requires GRANTs on performance_schema.session_variables.

tests/system/intermediate-master-replica-failure/01-relocate/expect_output

@@ -0,0 +1,6 @@
+127.0.0.1:10113<127.0.0.1:10112
+127.0.0.1:10114<127.0.0.1:10113
+127.0.0.1:10111      |0s|ok
++ 127.0.0.1:10112    |0s|ok
+  + 127.0.0.1:10113  |0s|ok
+    + 127.0.0.1:10114|0s|ok

tests/system/intermediate-master-replica-failure/01-relocate/run

@@ -0,0 +1,5 @@
+# 10111 -> 10112 -> 10113 -> 10114
+orchestrator-client -c relocate -i 127.0.0.1:10113 -d 127.0.0.1:10112
+orchestrator-client -c relocate -i 127.0.0.1:10114 -d 127.0.0.1:10113
+
+orchestrator-client -c topology-tabulated -alias ci | cut -d'|' -f 1,2,3

tests/system/intermediate-master-replica-failure/02-down-slave-3/run

@@ -0,0 +1 @@
+orchestrator-client -c replication-analysis

tests/system/intermediate-master-replica-failure/02-down-slave-3/setup

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+mysqladmin -uci -pci -h 127.0.0.1 --port=10114 shutdown
+sleep 20

tests/system/test.sh

@@ -187,7 +187,7 @@ test_single() {
   fi
 
   # test steps:
-  find "$tests_path/$test_name" ! -path . -type d -mindepth 1 -maxdepth 1 | sort | cut -d "/" -f 5 | while read test_step_name ; do
+  find "$tests_path/$test_name" -mindepth 1 -maxdepth 1 ! -path . -type d | sort | cut -d "/" -f 5 | while read test_step_name ; do
     [ "$test_step_name" == "." ] && continue
     test_step "$tests_path/$test_name/$test_step_name" "$test_name" "$test_step_name"
     if [ $? -ne 0 ] ; then
@@ -301,7 +301,7 @@ test_all() {
   while [ -s $tests_todo_file ] ; do
     echo -n > $tests_todo_file
 
-    find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | xargs ls -td1 | cut -d "/" -f 4 | egrep "$test_pattern" | while read test_name ; do
+    find $tests_path -mindepth 1 -maxdepth 1 ! -path . -type d | xargs ls -td1 | cut -d "/" -f 4 | egrep "$test_pattern" | while read test_name ; do
       if ! test_listed_as_attempted "$test_name" ; then
         echo "$test_name" >> $tests_todo_file
       fi
@@ -318,7 +318,7 @@ test_all() {
       fi
     done || return 1
   done
-  find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | xargs ls -td1 | cut -d "/" -f 4 | egrep "$test_pattern" | while read test_name ; do
+  find $tests_path -mindepth 1 -maxdepth 1 ! -path . -type d | xargs ls -td1 | cut -d "/" -f 4 | egrep "$test_pattern" | while read test_name ; do
     if ! test_listed_as_attempted "$test_name" ; then
       echo "# ERROR: tests completed by $test_name seems to have been skipped"
       exit 1