Add command to custom artifact dependencies (#2095)

* Add command to custom artifact dependencies

Allow users to specify a command to execute to determine dependencies
for a custom artifact. Dependencies must be returned as a valid JSON
array.

* Update docs to explain dependency command addition

* Update unit test so it passes on windows

docs/content/en/docs/how-tos/builders/_index.md

@@ -254,11 +254,63 @@ Currently, this only works with the build type `local`. Supported schema for `cu
 
 
 `buildCommand` is *required* and points skaffold to the custom build script which will be executed to build the artifact.
+
+#### Dependencies for a Custom Artifact
+
 `dependencies` tells the skaffold file watcher which files should be watched to trigger rebuilds and file syncs.  Supported schema for `dependencies` includes:
 
 
 {{< schema root="CustomDependencies" >}}
 
+##### Paths and Ignore
+`Paths` and `Ignore` are arrays used to list dependencies. 
+Any paths in `Ignore` will be ignored by the skaffold file watcher, even if they are also specified in `Paths`.
+`Ignore` will only work in conjunction with `Paths`, and with none of the other custom artifact dependency types.
+
+```yaml
+custom:
+  buildCommand: ./build.sh
+  dependencies:
+    paths:
+    - pkg/**
+    - src/*.go
+    ignore:
+    - vendor/**
+```
+
+
+##### Dockerfile
+Skaffold can calculate dependencies from a Dockerfile for a custom artifact.
+Passing in the path to the Dockerfile and any build args, if necessary, will allow skaffold to do dependency calculation.
+
+{{< schema root="DockerfileDependency" >}}
+
+```yaml
+custom:
+  buildCommand: ./build.sh
+  dependencies:
+    dockerfile:
+      path: path/to/Dockerfile
+      buildArgs:
+        file: foo
+```
+
+##### Getting depedencies from a command
+Sometimes you might have a builder that can provide the dependencies for a given artifact.
+For example bazel has the `bazel query deps` command.
+Custom artifact builders can ask Skaffold to execute a custom command, which Skaffold can use to get the dependencies for the artifact for file watching.
+
+The command *must* return dependencies as a JSON array, otherwise skaffold will error out.
+
+For example, the following configuration is valid, as executing the dependency command returns a valid JSON array.
+
+```yaml
+custom:
+  buildCommand: ./build.sh
+  dependencies:
+    command: echo ["file1","file2","file3"]
+```
+
 #### Custom Build Scripts and File Sync
 Syncable files must be included in both the `paths` section of `dependencies`, so that the skaffold file watcher knows to watch them, and the `sync` section, so that skaffold knows to sync them.  
 

docs/content/en/schemas/v1beta10.json

@@ -546,6 +546,11 @@
     },
     "CustomDependencies": {
       "properties": {
+        "command": {
+          "type": "string",
+          "description": "represents a custom command that skaffold executes to obtain dependencies. The output of this command *must* be a valid JSON array.",
+          "x-intellij-html-description": "represents a custom command that skaffold executes to obtain dependencies. The output of this command <em>must</em> be a valid JSON array."
+        },
         "dockerfile": {
           "$ref": "#/definitions/DockerfileDependency",
           "description": "should be set if the artifact is built from a Dockerfile, from which skaffold can determine dependencies.",
@@ -572,6 +577,7 @@
       },
       "preferredOrder": [
         "dockerfile",
+        "command",
         "paths",
         "ignore"
       ],

pkg/skaffold/build/custom/dependencies.go

@@ -18,10 +18,14 @@ package custom
 
 import (
 	"context"
+	"encoding/json"
+	"os/exec"
 	"sort"
+	"strings"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 	"github.com/pkg/errors"
 )
 
@@ -33,6 +37,19 @@ func GetDependencies(ctx context.Context, workspace string, a *latest.CustomArti
 		dockerfile := a.Dependencies.Dockerfile
 		return docker.GetDependencies(ctx, workspace, dockerfile.Path, dockerfile.BuildArgs, insecureRegistries)
 
+	case a.Dependencies.Command != "":
+		split := strings.Split(a.Dependencies.Command, " ")
+		cmd := exec.CommandContext(ctx, split[0], split[1:]...)
+		output, err := util.RunCmdOut(cmd)
+		if err != nil {
+			return nil, errors.Wrapf(err, "getting dependencies from command: %s", a.Dependencies.Command)
+		}
+		var deps []string
+		if err := json.Unmarshal(output, &deps); err != nil {
+			return nil, errors.Wrap(err, "unmarshalling dependency output into string array")
+		}
+		return deps, nil
+
 	default:
 		files, err := docker.WalkWorkspace(workspace, a.Dependencies.Ignore, a.Dependencies.Paths)
 		if err != nil {

pkg/skaffold/build/custom/dependencies_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 	"github.com/GoogleContainerTools/skaffold/testutil"
 )
 
@@ -58,6 +59,26 @@ func TestGetDependenciesDockerfile(t *testing.T) {
 	testutil.CheckErrorAndDeepEqual(t, false, err, expected, deps)
 }
 
+func TestGetDependenciesCommand(t *testing.T) {
+
+	defer func(c util.Command) { util.DefaultExecCommand = c }(util.DefaultExecCommand)
+	util.DefaultExecCommand = testutil.NewFakeCmd(t).WithRunOut(
+		"echo [\"file1\",\"file2\",\"file3\"]",
+		"[\"file1\",\"file2\",\"file3\"]",
+	)
+
+	customArtifact := &latest.CustomArtifact{
+		Dependencies: &latest.CustomDependencies{
+			Command: "echo [\"file1\",\"file2\",\"file3\"]",
+		},
+	}
+
+	expected := []string{"file1", "file2", "file3"}
+	deps, err := GetDependencies(context.Background(), "", customArtifact, nil)
+
+	testutil.CheckErrorAndDeepEqual(t, false, err, expected, deps)
+}
+
 func TestGetDependenciesPaths(t *testing.T) {
 	tmpDir, cleanup := testutil.NewTempDir(t)
 	defer cleanup()

pkg/skaffold/schema/latest/config.go

@@ -591,6 +591,8 @@ type CustomArtifact struct {
 type CustomDependencies struct {
 	// Dockerfile should be set if the artifact is built from a Dockerfile, from which skaffold can determine dependencies.
 	Dockerfile *DockerfileDependency `yaml:"dockerfile,omitempty" yamltags:"oneOf=dependency"`
+	// Command represents a custom command that skaffold executes to obtain dependencies. The output of this command *must* be a valid JSON array.
+	Command string `yaml:"command,omitempty" yamltags:"oneOf=dependency"`
 	// Paths should be set to the file dependencies for this artifact, so that the skaffold file watcher knows when to rebuild and perform file synchronization.
 	Paths []string `yaml:"paths,omitempty" yamltags:"oneOf=dependency"`
 	// Ignore specifies the paths that should be ignored by skaffold's file watcher. If a file exists in both `paths` and in `ignore`, it will be ignored, and will be excluded from both rebuilds and file synchronization.

pkg/skaffold/schema/validation/validation.go

@@ -72,7 +72,7 @@ func validateCustomDependencies(artifacts []*latest.Artifact) (errs []error) {
 		if a.CustomArtifact.Dependencies.Ignore == nil {
 			continue
 		}
-		if a.CustomArtifact.Dependencies.Dockerfile != nil {
+		if a.CustomArtifact.Dependencies.Dockerfile != nil || a.CustomArtifact.Dependencies.Command != "" {
 			errs = append(errs, fmt.Errorf("artifact %s has invalid dependencies; dependencies.ignore can only be used in conjunction with dependencies.paths", a.ImageName))
 		}
 	}

pkg/skaffold/schema/validation/validation_test.go

@@ -500,6 +500,13 @@ func TestValidateCustomDependencies(t *testing.T) {
 				Ignore: []string{"ignoreme"},
 			},
 			expectedErrors: 1,
+		}, {
+			description: "ignore in conjunction with command",
+			dependencies: &latest.CustomDependencies{
+				Command: "bazel query deps",
+				Ignore:  []string{"ignoreme"},
+			},
+			expectedErrors: 1,
 		},
 	}