diff --git a/cmd/hope/deploy.go b/cmd/hope/deploy.go index c03e4e4..ad8d404 100644 --- a/cmd/hope/deploy.go +++ b/cmd/hope/deploy.go @@ -52,15 +52,18 @@ var deployCmd = &cobra.Command{ return nil } - // Do a pass over the resources, and make sure that there's a docker - // build step before potentially asking the user to type in their - // password to elevate + // Do a pass over the resources to be deployed, and determine what + // kinds of local operations need to be done before all of these + // things can be deployed. hasDockerResource := false + hasKubernetesResource := false for _, resource := range *resources { resourceType, _ := resource.GetType() - if resourceType == ResourceTypeDockerBuild { + switch resourceType { + case ResourceTypeDockerBuild: hasDockerResource = true - break + case ResourceTypeFile, ResourceTypeInline, ResourceTypeJob, ResourceTypeExec: + hasKubernetesResource = true } } @@ -75,22 +78,22 @@ var deployCmd = &cobra.Command{ } } - // Wait as long as possible before pulling the temporary kubectl from - // a master node. - // TODO: Implement something similar to the hasDockerResource process - // above; if there isn't anything that needs to talk to kubernetes, - // don't even bother pulling the kubeconfig. - masters := viper.GetStringSlice("masters") - kubectl, err := kubeutil.NewKubectlFromAnyNode(masters) - if err != nil { - return err - } + var kubectl *kubeutil.Kubectl + if hasKubernetesResource { + masters := viper.GetStringSlice("masters") - defer kubectl.Destroy() + var err error + kubectl, err = kubeutil.NewKubectlFromAnyNode(masters) + if err != nil { + return err + } + + defer kubectl.Destroy() + } // TODO: Should be done in hope pkg // TODO: Add validation to ensure each type of deployment can run given - // the current dev environment -- ensure docker is can connect, etc. + // the current dev environment -- ensure docker can connect, etc. for _, resource := range *resources { log.Debug("Starting deployment of ", resource.Name) resourceType, err := resource.GetType() @@ -135,30 +138,59 @@ var deployCmd = &cobra.Command{ return err } case ResourceTypeDockerBuild: - // Strip the actual tag off the repo so that it defaults to the - // latest. - tagSeparator := strings.LastIndex(resource.Build.Tag, ":") - pullImage := resource.Build.Tag - if tagSeparator != -1 { - pullImage = pullImage[:tagSeparator] + isCacheCommand := len(resource.Build.Source) != 0 + isBuildCommand := len(resource.Build.Path) != 0 + + if isCacheCommand && isBuildCommand { + return errors.New(fmt.Sprintf("Docker build step %s cannot have a path and a source", resource.Name)) + } + + // TODO: Move these to constants somewhere + pullConstraintAlways := resource.Build.Pull == "always" + pullConstraintIfNotPresent := resource.Build.Pull == "if-not-present" || resource.Build.Pull == "" + + if !pullConstraintAlways && !pullConstraintIfNotPresent { + return errors.New(fmt.Sprintf("Unknown Docker image pull constraint: %s", resource.Build.Pull)) + } + + pullImage := "" + if isCacheCommand { + pullImage = resource.Build.Source + } else { + pullImage = resource.Build.Tag + } + + ifNotPresentShouldPull := false + if pullConstraintIfNotPresent { + output, err := docker.GetDocker("images", pullImage) + if err != nil { + return err + } + + if len(strings.Split(output, "\n")) == 1 { + log.Info(fmt.Sprintf("Docker image %s not found locally, must pull from upstream.", pullImage)) + ifNotPresentShouldPull = true + } else { + log.Debug(fmt.Sprintf("Docker image %s found, skipping upstream pull")) + } } - if err := docker.ExecDocker("pull", pullImage); err != nil { - // Maybe the image was pushed with the given tag. - // Maybe the tag is something like :stable. - // Hopefully we can grab a few layers at least. - if err := docker.ExecDocker("pull", resource.Build.Tag); err != nil { - log.Warn("Failed to pull existing images for ", pullImage, ". Maybe this image doesn't exist?") - - // Don't return any errors here. - // If this is the first time this image is being - // pushed, there will be nothing to pull, and - // this will never succeed. + if ifNotPresentShouldPull || pullConstraintAlways { + if err := docker.ExecDocker("pull", pullImage); err != nil { + return errors.New(fmt.Sprintf("Failed to find image named %s", pullImage)) } } - if err := docker.ExecDocker("build", resource.Build.Path, "-t", resource.Build.Tag); err != nil { - return err + + if isBuildCommand { + if err := docker.ExecDocker("build", resource.Build.Path, "-t", resource.Build.Tag); err != nil { + return err + } + } else { + if err := docker.ExecDocker("tag", resource.Build.Source, resource.Build.Tag); err != nil { + return err + } } + if err := docker.ExecDocker("push", resource.Build.Tag); err != nil { return err } diff --git a/cmd/hope/root.go b/cmd/hope/root.go index ad59780..dd3427d 100644 --- a/cmd/hope/root.go +++ b/cmd/hope/root.go @@ -155,6 +155,16 @@ func patchInvocations() { return oldExecDocker(args...) } + oldGetDocker := docker.GetDocker + docker.GetDocker = func(args ...string) (string, error) { + if docker.UseSudo { + log.Debug("sudo docker ", strings.Join(args, " ")) + } else { + log.Debug("docker ", strings.Join(args, " ")) + } + return oldGetDocker(args...) + } + oldEnvsubstBytes := envsubst.GetEnvsubstBytes envsubst.GetEnvsubstBytes = func(args []string, contents []byte) ([]byte, error) { argsKeys := []string{} diff --git a/cmd/hope/utils.go b/cmd/hope/utils.go index 6691330..9e449cd 100644 --- a/cmd/hope/utils.go +++ b/cmd/hope/utils.go @@ -25,8 +25,10 @@ const ( // Should be defined in hope pkg type BuildSpec struct { - Path string - Tag string + Path string + Source string + Tag string + Pull string } type ExecSpec struct { @@ -62,7 +64,7 @@ func (resource *Resource) GetType() (string, error) { if len(resource.Inline) != 0 { detectedTypes = append(detectedTypes, ResourceTypeInline) } - if len(resource.Build.Path) != 0 && len(resource.Build.Tag) != 0 { + if (len(resource.Build.Path) != 0 || len(resource.Build.Source) != 0) && len(resource.Build.Tag) != 0 { detectedTypes = append(detectedTypes, ResourceTypeDockerBuild) } if len(resource.Job) != 0 { diff --git a/hope.yaml b/hope.yaml index ee8e7f2..0db3330 100644 --- a/hope.yaml +++ b/hope.yaml @@ -63,16 +63,29 @@ resources: # Build and push a docker image to the registry. # Doesn't include a kubectl command at all, so that can be done in a step # after a step like this appears. - # Hope will pull the latest image available from the registry before building - # it in an attempt to save some compute time rebuilding layers that haven't - # changed since the last deployment. + # Hope will optionally try to pull the image from the registry before + # building it in an attempt to save some compute time rebuilding layers + # that haven't changed since the last push. + # Pull from source registry can be done in similar ways to Kubernetes' + # Always and IfNotPresent with the "always", if "if-not-present" values. + # Like Kubernetes, defaults to "if-not-present". # Because docker builds tend to require a bit more state in them, providing a # local path is all that's currently supported. + # Now that Docker Hub has rolled out rate limits on their APIs, a Docker + # build step also has the option to just copy an existing source tag, and + # push it to the local registry. - name: build-some-image build: path: some-dir-with-dockerfile + pull: always tag: registry.internal.aleemhaji.com/example-repo:latest tags: [app1] + - name: copy-some-image + build: + source: python:3.7 + pull: if-not-present + tag: registry.internal.aleemhaji.com/python:3.7 + tags: [dockercache] # When a spec comes with an initialization procedure, a job type can be used. # These will wait until the job with the specified name is completed. # If the job fails, the deployment stops so that other resources that may diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index 29014f0..7d6c632 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -15,6 +15,7 @@ var UseSudo bool = false // is a lot nicer than // docker.ExecDocker("pull", ...) type ExecDockerFunc func(args ...string) error +type GetDockerFunc func(args ...string) (string, error) var ExecDocker ExecDockerFunc = func(args ...string) error { var osCmd *exec.Cmd @@ -31,6 +32,21 @@ var ExecDocker ExecDockerFunc = func(args ...string) error { return osCmd.Run() } +var GetDocker GetDockerFunc = func(args ...string) (string, error) { + var osCmd *exec.Cmd + if UseSudo { + allArgs := append([]string{"docker"}, args...) + osCmd = exec.Command("sudo", allArgs...) + } else { + osCmd = exec.Command("docker", args...) + } + osCmd.Stdin = os.Stdin + osCmd.Stderr = os.Stderr + + outputBytes, err := osCmd.Output() + return string(outputBytes), err +} + func SetUseSudo() { osCmd := exec.Command("docker", "ps")