Adds the ability to directly self-rehost docker images (#39)

* Adds the ability to rehost docker images in the local registry.

* Changes pull behaviour to be more clear.

* comment cleanup

* Actually fall back to if-not-present properly

* whitespace tightening

cmd/hope/deploy.go

@@ -52,15 +52,18 @@ var deployCmd = &cobra.Command{
 			return nil
 		}
 
-		// Do a pass over the resources, and make sure that there's a docker
-		//   build step before potentially asking the user to type in their
-		//   password to elevate
+		// Do a pass over the resources to be deployed, and determine what
+		//   kinds of local operations need to be done before all of these
+		//   things can be deployed.
 		hasDockerResource := false
+		hasKubernetesResource := false
 		for _, resource := range *resources {
 			resourceType, _ := resource.GetType()
-			if resourceType == ResourceTypeDockerBuild {
+			switch resourceType {
+			case ResourceTypeDockerBuild:
 				hasDockerResource = true
-				break
+			case ResourceTypeFile, ResourceTypeInline, ResourceTypeJob, ResourceTypeExec:
+				hasKubernetesResource = true
 			}
 		}
 
@@ -75,22 +78,22 @@ var deployCmd = &cobra.Command{
 			}
 		}
 
-		// Wait as long as possible before pulling the temporary kubectl from
-		//   a master node.
-		// TODO: Implement something similar to the hasDockerResource process
-		//   above; if there isn't anything that needs to talk to kubernetes,
-		//   don't even bother pulling the kubeconfig.
-		masters := viper.GetStringSlice("masters")
-		kubectl, err := kubeutil.NewKubectlFromAnyNode(masters)
-		if err != nil {
-			return err
-		}
+		var kubectl *kubeutil.Kubectl
+		if hasKubernetesResource {
+			masters := viper.GetStringSlice("masters")
 
-		defer kubectl.Destroy()
+			var err error
+			kubectl, err = kubeutil.NewKubectlFromAnyNode(masters)
+			if err != nil {
+				return err
+			}
+
+			defer kubectl.Destroy()
+		}
 
 		// TODO: Should be done in hope pkg
 		// TODO: Add validation to ensure each type of deployment can run given
-		//   the current dev environment -- ensure docker is can connect, etc.
+		//   the current dev environment -- ensure docker can connect, etc.
 		for _, resource := range *resources {
 			log.Debug("Starting deployment of ", resource.Name)
 			resourceType, err := resource.GetType()
@@ -135,30 +138,59 @@ var deployCmd = &cobra.Command{
 					return err
 				}
 			case ResourceTypeDockerBuild:
-				// Strip the actual tag off the repo so that it defaults to the
-				//   latest.
-				tagSeparator := strings.LastIndex(resource.Build.Tag, ":")
-				pullImage := resource.Build.Tag
-				if tagSeparator != -1 {
-					pullImage = pullImage[:tagSeparator]
+				isCacheCommand := len(resource.Build.Source) != 0
+				isBuildCommand := len(resource.Build.Path) != 0
+
+				if isCacheCommand && isBuildCommand {
+					return errors.New(fmt.Sprintf("Docker build step %s cannot have a path and a source", resource.Name))
+				}
+
+				// TODO: Move these to constants somewhere
+				pullConstraintAlways := resource.Build.Pull == "always"
+				pullConstraintIfNotPresent := resource.Build.Pull == "if-not-present" || resource.Build.Pull == ""
+
+				if !pullConstraintAlways && !pullConstraintIfNotPresent {
+					return errors.New(fmt.Sprintf("Unknown Docker image pull constraint: %s", resource.Build.Pull))
+				}
+
+				pullImage := ""
+				if isCacheCommand {
+					pullImage = resource.Build.Source
+				} else {
+					pullImage = resource.Build.Tag
+				}
+
+				ifNotPresentShouldPull := false
+				if pullConstraintIfNotPresent {
+					output, err := docker.GetDocker("images", pullImage)
+					if err != nil {
+						return err
+					}
+
+					if len(strings.Split(output, "\n")) == 1 {
+						log.Info(fmt.Sprintf("Docker image %s not found locally, must pull from upstream.", pullImage))
+						ifNotPresentShouldPull = true
+					} else {
+						log.Debug(fmt.Sprintf("Docker image %s found, skipping upstream pull"))
+					}
 				}
 
-				if err := docker.ExecDocker("pull", pullImage); err != nil {
-					// Maybe the image was pushed with the given tag.
-					// Maybe the tag is something like :stable.
-					// Hopefully we can grab a few layers at least.
-					if err := docker.ExecDocker("pull", resource.Build.Tag); err != nil {
-						log.Warn("Failed to pull existing images for ", pullImage, ". Maybe this image doesn't exist?")
-
-						// Don't return any errors here.
-						// If this is the first time this image is being
-						//   pushed, there will be nothing to pull, and
-						//   this will never succeed.
+				if ifNotPresentShouldPull || pullConstraintAlways {
+					if err := docker.ExecDocker("pull", pullImage); err != nil {
+						return errors.New(fmt.Sprintf("Failed to find image named %s", pullImage))
 					}
 				}
-				if err := docker.ExecDocker("build", resource.Build.Path, "-t", resource.Build.Tag); err != nil {
-					return err
+
+				if isBuildCommand {
+					if err := docker.ExecDocker("build", resource.Build.Path, "-t", resource.Build.Tag); err != nil {
+						return err
+					}
+				} else {
+					if err := docker.ExecDocker("tag", resource.Build.Source, resource.Build.Tag); err != nil {
+						return err
+					}
 				}
+
 				if err := docker.ExecDocker("push", resource.Build.Tag); err != nil {
 					return err
 				}

cmd/hope/root.go

@@ -155,6 +155,16 @@ func patchInvocations() {
 		return oldExecDocker(args...)
 	}
 
+	oldGetDocker := docker.GetDocker
+	docker.GetDocker = func(args ...string) (string, error) {
+		if docker.UseSudo {
+			log.Debug("sudo docker ", strings.Join(args, " "))
+		} else {
+			log.Debug("docker ", strings.Join(args, " "))
+		}
+		return oldGetDocker(args...)
+	}
+
 	oldEnvsubstBytes := envsubst.GetEnvsubstBytes
 	envsubst.GetEnvsubstBytes = func(args []string, contents []byte) ([]byte, error) {
 		argsKeys := []string{}

cmd/hope/utils.go

@@ -25,8 +25,10 @@ const (
 
 // Should be defined in hope pkg
 type BuildSpec struct {
-	Path string
-	Tag  string
+	Path   string
+	Source string
+	Tag    string
+	Pull   string
 }
 
 type ExecSpec struct {
@@ -62,7 +64,7 @@ func (resource *Resource) GetType() (string, error) {
 	if len(resource.Inline) != 0 {
 		detectedTypes = append(detectedTypes, ResourceTypeInline)
 	}
-	if len(resource.Build.Path) != 0 && len(resource.Build.Tag) != 0 {
+	if (len(resource.Build.Path) != 0 || len(resource.Build.Source) != 0) && len(resource.Build.Tag) != 0 {
 		detectedTypes = append(detectedTypes, ResourceTypeDockerBuild)
 	}
 	if len(resource.Job) != 0 {

hope.yaml

@@ -63,16 +63,29 @@ resources:
   # Build and push a docker image to the registry.
   # Doesn't include a kubectl command at all, so that can be done in a step
   #   after a step like this appears.
-  # Hope will pull the latest image available from the registry before building
-  #   it in an attempt to save some compute time rebuilding layers that haven't
-  #   changed since the last deployment.
+  # Hope will optionally try to pull the image from the registry before
+  #   building it in an attempt to save some compute time rebuilding layers 
+  #   that haven't changed since the last push.
+  # Pull from source registry can be done in similar ways to Kubernetes'
+  #   Always and IfNotPresent with the "always", if "if-not-present" values.
+  # Like Kubernetes, defaults to "if-not-present".
   # Because docker builds tend to require a bit more state in them, providing a
   #   local path is all that's currently supported.
+  # Now that Docker Hub has rolled out rate limits on their APIs, a Docker
+  #   build step also has the option to just copy an existing source tag, and
+  #   push it to the local registry.
   - name: build-some-image
     build:
       path: some-dir-with-dockerfile
+      pull: always
       tag: registry.internal.aleemhaji.com/example-repo:latest
     tags: [app1]
+  - name: copy-some-image
+    build:
+      source: python:3.7
+      pull: if-not-present
+      tag: registry.internal.aleemhaji.com/python:3.7
+    tags: [dockercache]
   # When a spec comes with an initialization procedure, a job type can be used.
   # These will wait until the job with the specified name is completed.
   # If the job fails, the deployment stops so that other resources that may

pkg/docker/docker.go

@@ -15,6 +15,7 @@ var UseSudo bool = false
 // is a lot nicer than
 //   docker.ExecDocker("pull", ...)
 type ExecDockerFunc func(args ...string) error
+type GetDockerFunc func(args ...string) (string, error)
 
 var ExecDocker ExecDockerFunc = func(args ...string) error {
 	var osCmd *exec.Cmd
@@ -31,6 +32,21 @@ var ExecDocker ExecDockerFunc = func(args ...string) error {
 	return osCmd.Run()
 }
 
+var GetDocker GetDockerFunc = func(args ...string) (string, error) {
+	var osCmd *exec.Cmd
+	if UseSudo {
+		allArgs := append([]string{"docker"}, args...)
+		osCmd = exec.Command("sudo", allArgs...)
+	} else {
+		osCmd = exec.Command("docker", args...)
+	}
+	osCmd.Stdin = os.Stdin
+	osCmd.Stderr = os.Stderr
+
+	outputBytes, err := osCmd.Output()
+	return string(outputBytes), err
+}
+
 func SetUseSudo() {
 	osCmd := exec.Command("docker", "ps")