Fix etcdbackup error reporting

rancher · Jan 24, 2019 · c229a06 · c229a06
1 parent c73a58d
commit c229a06
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 8 deletions.
diff --git a/docker/docker.go b/docker/docker.go
@@ -485,3 +485,17 @@ func DoRestartContainer(ctx context.Context, dClient *client.Client, containerNa
 	log.Infof(ctx, "[restart/%s] Successfully restarted container on host [%s]", containerName, hostname)
 	return nil
 }
+
+func GetContainerOutput(ctx context.Context, dClient *client.Client, containerName, hostname string) (int64, string, string, error) {
+	status, err := WaitForContainer(ctx, dClient, hostname, containerName)
+	if err != nil {
+		return 1, "", "", err
+	}
+
+	stderr, stdout, err := GetContainerLogsStdoutStderr(ctx, dClient, containerName, "1", false)
+	if err != nil {
+		return 1, "", "", err
+	}
+
+	return status, stdout, stderr, nil
+}
diff --git a/services/etcd.go b/services/etcd.go
@@ -290,16 +290,20 @@ func RunEtcdSnapshotSave(ctx context.Context, etcdHost *hosts.Host, prsMap map[s
 		if err := docker.DoRunContainer(ctx, etcdHost.DClient, imageCfg, hostCfg, EtcdSnapshotOnceContainerName, etcdHost.Address, ETCDRole, prsMap); err != nil {
 			return err
 		}
-		status, err := docker.WaitForContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName)
+		status, _, stderr, err := docker.GetContainerOutput(ctx, etcdHost.DClient, EtcdSnapshotOnceContainerName, etcdHost.Address)
 		if status != 0 || err != nil {
-			err := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName)
+			if removeErr := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName); removeErr != nil {
+				log.Warnf(ctx, "Failed to remove container [%s]: %v", removeErr)
+			}
 			if err != nil {
-				return fmt.Errorf("Failed to take etcd snapshot exit code [%d], failed to exit container [%s]: %v ", status, EtcdDownloadBackupContainerName, err)
+				return err
 			}
-			return fmt.Errorf("Failed to take etcd snapshot exit code [%d]: %v", status, err)
+			return fmt.Errorf("Failed to take one-time snapshot, exit code [%d]: %v", status, stderr)
 		}
+
 		return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdSnapshotOnceContainerName)
 	}
+
 	if err := docker.DoRunContainer(ctx, etcdHost.DClient, imageCfg, hostCfg, EtcdSnapshotContainerName, etcdHost.Address, ETCDRole, prsMap); err != nil {
 		return err
 	}
@@ -349,12 +353,16 @@ func DownloadEtcdSnapshot(ctx context.Context, etcdHost *hosts.Host, prsMap map[
 	if err := docker.DoRunContainer(ctx, etcdHost.DClient, imageCfg, hostCfg, EtcdDownloadBackupContainerName, etcdHost.Address, ETCDRole, prsMap); err != nil {
 		return err
 	}
-	status, err := docker.WaitForContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdDownloadBackupContainerName)
+
+	status, _, stderr, err := docker.GetContainerOutput(ctx, etcdHost.DClient, EtcdDownloadBackupContainerName, etcdHost.Address)
 	if status != 0 || err != nil {
-		if err2 := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdDownloadBackupContainerName); err2 != nil {
-			log.Warnf(ctx, "Failed to delete backup download container [%s]: %v", EtcdDownloadBackupContainerName, err2)
+		if removeErr := docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdDownloadBackupContainerName); removeErr != nil {
+			log.Warnf(ctx, "Failed to remove container [%s]: %v", removeErr)
+		}
+		if err != nil {
+			return err
 		}
-		return fmt.Errorf("Failed to download etcd snapshot from s3, exit code [%d]: %v", status, err)
+		return fmt.Errorf("Failed to download etcd snapshot from s3, exit code [%d]: %v", status, stderr)
 	}
 	return docker.RemoveContainer(ctx, etcdHost.DClient, etcdHost.Address, EtcdDownloadBackupContainerName)
 }