Create 'link' secrets for BOSH links in deployment

[#169418463](https://www.pivotaltracker.com/story/show/169418463)

.dockerignore

@@ -1,5 +1,11 @@
 binaries/
 build/
+helm/
 tmp/
 
+deploy/
+docs/
+e2e/
+integration/
+testing/
 Dockerfile

bin/include/dependencies

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-quarks_job_release="v0.0.0-0.gb5d55d1"
+git_sha="23b0d08"
+quarks_job_release="v0.0.0-0.g$git_sha"
 
 # QUARKS_JOB_IMAGE_TAG is used for integration tests
 if [ -z ${QUARKS_JOB_IMAGE_TAG+x} ]; then

cmd/internal/bpm_configs.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"sigs.k8s.io/yaml"
@@ -85,26 +86,26 @@ instance group.
 			return errors.Wrapf(err, "%s Loading bosh manifest file failed. Please check the file contents and try again.", bpmFailedMessage)
 		}
 
-		dg, err := manifest.NewInstanceGroupResolver(baseDir, *m, instanceGroupName)
+		igr, err := manifest.NewInstanceGroupResolver(afero.NewOsFs(), baseDir, *m, instanceGroupName)
 		if err != nil {
 			return errors.Wrap(err, bpmFailedMessage)
 		}
 
-		bpmInfo, err := dg.BPMInfo()
+		bpmInfo, err := igr.BPMInfo()
 		if err != nil {
 			return errors.Wrap(err, bpmFailedMessage)
 		}
 
 		bpmBytes, err := yaml.Marshal(bpmInfo)
 		if err != nil {
-			return errors.Wrapf(err, "%s YAML marshalling bpmConfigs spec returned by dg.BPMConfigs() failed.", bpmFailedMessage)
+			return errors.Wrapf(err, "%s YAML marshalling bpmConfigs spec returned by igr.BPMConfigs() failed.", bpmFailedMessage)
 		}
 
 		jsonBytes, err := json.Marshal(map[string]string{
 			"bpm.yaml": string(bpmBytes),
 		})
 		if err != nil {
-			return errors.Wrapf(err, "%s JSON marshalling bpmConfigs spec returned by dg.BPMConfigs() failed.", bpmFailedMessage)
+			return errors.Wrapf(err, "%s JSON marshalling bpmConfigs spec returned by igr.BPMConfigs() failed.", bpmFailedMessage)
 		}
 
 		err = ioutil.WriteFile(outputFilePath, jsonBytes, 0644)

cmd/internal/instance_group.go

@@ -3,17 +3,19 @@ package cmd
 import (
 	"encoding/json"
 	"io/ioutil"
+	"path/filepath"
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
 	"code.cloudfoundry.org/cf-operator/pkg/bosh/manifest"
 	"code.cloudfoundry.org/quarks-utils/pkg/cmd"
 )
 
-const dGatherFailedMessage = "instance-group command failed."
+const igFailedMessage = "instance-group command failed."
 
 // instanceGroupCmd command to create an instance group manifest where all
 // properties are resolved
@@ -44,65 +46,70 @@ This will resolve the properties of an instance group and return a manifest for
 
 		boshManifestPath, err := boshManifestFlagValidation()
 		if err != nil {
-			return errors.Wrap(err, dGatherFailedMessage)
+			return errors.Wrap(err, igFailedMessage)
 		}
 
 		baseDir, err := baseDirFlagValidation()
 		if err != nil {
-			return errors.Wrap(err, dGatherFailedMessage)
+			return errors.Wrap(err, igFailedMessage)
 		}
 
 		namespace := viper.GetString("cf-operator-namespace")
 		if len(namespace) == 0 {
-			return errors.Errorf("%s cf-operator-namespace flag is empty.", dGatherFailedMessage)
+			return errors.Errorf("%s cf-operator-namespace flag is empty.", igFailedMessage)
 		}
 
 		outputFilePath, err := outputFilePathFlagValidation()
 		if err != nil {
-			return errors.Wrap(err, dGatherFailedMessage)
+			return errors.Wrap(err, igFailedMessage)
 		}
 
 		instanceGroupName, err := instanceGroupFlagValidation()
 		if err != nil {
-			return errors.Wrap(err, dGatherFailedMessage)
+			return errors.Wrap(err, igFailedMessage)
 		}
 
 		boshManifestBytes, err := ioutil.ReadFile(boshManifestPath)
 		if err != nil {
-			return errors.Wrapf(err, "%s Reading file specified in the bosh-manifest-path flag failed. Please check the filepath to continue.", dGatherFailedMessage)
+			return errors.Wrapf(err, "%s Reading file specified in the bosh-manifest-path flag failed. Please check the filepath to continue.", igFailedMessage)
 		}
 
 		m, err := manifest.LoadYAML(boshManifestBytes)
 		if err != nil {
-			return errors.Wrapf(err, "%s Loading bosh manifest file failed. Please check the file contents and try again.", dGatherFailedMessage)
+			return errors.Wrapf(err, "%s Loading bosh manifest file failed. Please check the file contents and try again.", igFailedMessage)
 		}
 
-		dg, err := manifest.NewInstanceGroupResolver(baseDir, *m, instanceGroupName)
+		igr, err := manifest.NewInstanceGroupResolver(afero.NewOsFs(), baseDir, *m, instanceGroupName)
 		if err != nil {
-			return errors.Wrap(err, dGatherFailedMessage)
+			return errors.Wrap(err, igFailedMessage)
 		}
 
-		manifest, err := dg.Manifest()
+		manifest, err := igr.Manifest()
 		if err != nil {
-			return errors.Wrap(err, dGatherFailedMessage)
+			return errors.Wrap(err, igFailedMessage)
+		}
+
+		err = igr.SaveLinks(filepath.Dir(outputFilePath))
+		if err != nil {
+			return errors.Wrapf(err, "%s failed to write link output to file.", igFailedMessage)
 		}
 
 		// write instance group manifest to output.json
 		propertiesBytes, err := manifest.Marshal()
 		if err != nil {
-			return errors.Wrapf(err, "%s YAML marshalling instance group manifest failed.", dGatherFailedMessage)
+			return errors.Wrapf(err, "%s YAML marshalling instance group manifest failed.", igFailedMessage)
 		}
 
 		jsonBytes, err := json.Marshal(map[string]string{
 			"properties.yaml": string(propertiesBytes),
 		})
 		if err != nil {
-			return errors.Wrapf(err, "%s JSON marshalling instance group manifest failed.", dGatherFailedMessage)
+			return errors.Wrapf(err, "%s JSON marshalling instance group manifest failed.", igFailedMessage)
 		}
 
 		err = ioutil.WriteFile(outputFilePath, jsonBytes, 0644)
 		if err != nil {
-			return errors.Wrapf(err, "%s Writing json into a output file failed.", dGatherFailedMessage)
+			return errors.Wrapf(err, "%s Writing json into a output file failed.", igFailedMessage)
 		}
 
 		return nil

cmd/internal/util.go

@@ -79,7 +79,7 @@ func outputFilePathFlagValidation() (string, error) {
 }
 
 func outputFilePathFlagCobraSet(pf *flag.FlagSet, argToEnv map[string]string) {
-	pf.StringP("output-file-path", "", "", "Path of the file to which json output is redirected.")
+	pf.StringP("output-file-path", "", "", "Path of the file to which json output is written.")
 	argToEnv["output-file-path"] = "OUTPUT_FILE_PATH"
 }
 

e2e/cli/cli_test.go

@@ -146,7 +146,7 @@ var _ = Describe("CLI", func() {
 			Eventually(session.Out).Should(Say(`Flags:
   -m, --bosh-manifest-path string   \(BOSH_MANIFEST_PATH\) path to the bosh manifest file
   -h, --help                        help for variable-interpolation
-      --output-file-path string     \(OUTPUT_FILE_PATH\) Path of the file to which json output is redirected.
+      --output-file-path string     \(OUTPUT_FILE_PATH\) Path of the file to which json output is written.
   -v, --variables-dir string        \(VARIABLES_DIR\) path to the variables dir`))
 		})
 
@@ -182,7 +182,7 @@ var _ = Describe("CLI", func() {
   -m, --bosh-manifest-path string    \(BOSH_MANIFEST_PATH\) path to the bosh manifest file
   -h, --help                         help for instance-group
   -g, --instance-group-name string   \(INSTANCE_GROUP_NAME\) name of the instance group for data gathering
-      --output-file-path string      \(OUTPUT_FILE_PATH\) Path of the file to which json output is redirected.`))
+      --output-file-path string      \(OUTPUT_FILE_PATH\) Path of the file to which json output is written.`))
 		})
 
 		It("accepts the bosh-manifest-path as a parameter", func() {
@@ -217,7 +217,7 @@ var _ = Describe("CLI", func() {
   -m, --bosh-manifest-path string    \(BOSH_MANIFEST_PATH\) path to the bosh manifest file
   -h, --help                         help for bpm-configs
   -g, --instance-group-name string   \(INSTANCE_GROUP_NAME\) name of the instance group for data gathering
-      --output-file-path string      \(OUTPUT_FILE_PATH\) Path of the file to which json output is redirected.`))
+      --output-file-path string      \(OUTPUT_FILE_PATH\) Path of the file to which json output is written.`))
 		})
 
 		It("accepts the bosh-manifest-path as a parameter", func() {

go.mod

@@ -1,7 +1,7 @@
 module code.cloudfoundry.org/cf-operator
 
 require (
-	code.cloudfoundry.org/quarks-job v0.0.0-20191128150518-727b38941b4f
+	code.cloudfoundry.org/quarks-job v0.0.0-20191129095316-23b0d08e8376
 	code.cloudfoundry.org/quarks-utils v0.0.0-20191114145816-f753780f6d4f
 	github.com/bmatcuk/doublestar v1.1.1 // indirect
 	github.com/charlievieth/fs v0.0.0-20170613215519-7dc373669fa1 // indirect

go.sum

@@ -1,6 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-code.cloudfoundry.org/quarks-job v0.0.0-20191128150518-727b38941b4f h1:8Fk4bK29ics50/7wSHUILcF/Qy5VvPYbq16gnOIx9xU=
-code.cloudfoundry.org/quarks-job v0.0.0-20191128150518-727b38941b4f/go.mod h1:E1WpDsi0+UbI/W5mDCD6CYzUTh8p2nFIAoc3o3kLffQ=
+code.cloudfoundry.org/quarks-job v0.0.0-20191129095316-23b0d08e8376 h1:CDtBZJJSH4HQSdCJGSb12qy0pZGQ2/VUnQckScs45rM=
+code.cloudfoundry.org/quarks-job v0.0.0-20191129095316-23b0d08e8376/go.mod h1:E1WpDsi0+UbI/W5mDCD6CYzUTh8p2nFIAoc3o3kLffQ=
 code.cloudfoundry.org/quarks-utils v0.0.0-20191114145816-f753780f6d4f h1:csZ/4q+RneCMG/peE4pl+9/yM69uMog7SH1Yibaw6oE=
 code.cloudfoundry.org/quarks-utils v0.0.0-20191114145816-f753780f6d4f/go.mod h1:H6fVNegFsTZqOU2Cggvue8X5vfAdeGDKemikmwc7RBc=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=

integration/bosh_links_test.go

@@ -0,0 +1,71 @@
+package integration_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+
+	bm "code.cloudfoundry.org/cf-operator/testing/boshmanifest"
+	"code.cloudfoundry.org/quarks-utils/testing/machine"
+)
+
+var _ = Describe("BOSHLinks", func() {
+	const (
+		manifestRef    = "manifest"
+		deploymentName = "test"
+	)
+
+	var (
+		tearDowns    []machine.TearDownFunc
+		boshManifest corev1.Secret
+	)
+
+	AfterEach(func() {
+		Expect(env.TearDownAll(tearDowns)).To(Succeed())
+	})
+
+	JustBeforeEach(func() {
+		tearDown, err := env.CreateSecret(env.Namespace, boshManifest)
+		Expect(err).NotTo(HaveOccurred())
+		tearDowns = append(tearDowns, tearDown)
+
+		_, tearDown, err = env.CreateBOSHDeployment(env.Namespace,
+			env.SecretBOSHDeployment(deploymentName, manifestRef))
+		Expect(err).NotTo(HaveOccurred())
+		tearDowns = append(tearDowns, tearDown)
+	})
+
+	Context("when deployment has implicit links only", func() {
+		BeforeEach(func() {
+			boshManifest = env.BOSHManifestSecret(manifestRef, bm.NatsSmall)
+		})
+
+		It("creates a secret for each link", func() {
+			By("waiting for secrets", func() {
+				secretName := "link-test-nats"
+				err := env.WaitForSecret(env.Namespace, secretName)
+				Expect(err).NotTo(HaveOccurred())
+				secret, err := env.GetSecret(env.Namespace, secretName)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(secret.Data).Should(HaveKeyWithValue("nats.nats", []byte("{\"nats\":{\"password\":\"changeme\",\"port\":4222,\"user\":\"admin\"}}")))
+			})
+		})
+	})
+
+	Context("when deployment has explicit links", func() {
+		BeforeEach(func() {
+			boshManifest = env.BOSHManifestSecret(manifestRef, bm.NatsSmallWithLinks)
+		})
+
+		It("creates a secret for each link", func() {
+			By("waiting for secrets", func() {
+				err := env.WaitForSecret(env.Namespace, "link-test-nats")
+				Expect(err).NotTo(HaveOccurred())
+				secret, err := env.GetSecret(env.Namespace, "link-test-nats")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(secret.Data).Should(HaveKeyWithValue("nats.nuts", []byte("{\"nats\":{\"password\":\"changeme\",\"port\":4222,\"user\":\"admin\"}}")))
+			})
+		})
+	})
+})

pkg/bosh/converter/job_factory.go

@@ -83,6 +83,7 @@ func (f *JobFactory) VariableInterpolationJob(manifest bdm.Manifest) (*qjv1a1.Qu
 	}
 
 	qJobName := fmt.Sprintf("dm-%s", manifest.Name)
+	secretName := names.DesiredManifestPrefix(manifest.Name) + VarInterpolationContainerName
 
 	// Construct the var interpolation auto-errand qJob
 	qJob := &qjv1a1.QuarksJob{
@@ -95,13 +96,14 @@ func (f *JobFactory) VariableInterpolationJob(manifest bdm.Manifest) (*qjv1a1.Qu
 		},
 		Spec: qjv1a1.QuarksJobSpec{
 			Output: &qjv1a1.Output{
-				NamePrefix: names.DesiredManifestPrefix(manifest.Name),
+				OutputMap: qjv1a1.OutputMap{
+					VarInterpolationContainerName: qjv1a1.NewFileToSecret("output.json", secretName, true),
+				},
 				SecretLabels: map[string]string{
 					bdv1.LabelDeploymentName:       manifest.Name,
 					bdv1.LabelDeploymentSecretType: names.DeploymentSecretTypeManifestWithOps.String(),
 					bdm.LabelReferencedJobName:     fmt.Sprintf("instance-group-%s", manifest.Name),
 				},
-				Versioned: true,
 			},
 			Trigger: qjv1a1.Trigger{
 				Strategy: qjv1a1.TriggerOnce,
@@ -160,10 +162,15 @@ func (f *JobFactory) InstanceGroupManifestJob(manifest bdm.Manifest) (*qjv1a1.Qu
 		namespace:    f.Namespace,
 	}
 
+	linkOutputs := map[string]string{}
+
 	for _, ig := range manifest.InstanceGroups {
 		if ig.Instances != 0 {
-			// One container per Instance Group
-			// There will be one secret generated for each of these containers
+			// Additional secret for BOSH links per instance group
+			containerName := names.Sanitize(ig.Name)
+			linkOutputs[containerName] = names.Sanitize(fmt.Sprintf("link-%s-%s", manifest.Name, ig.Name))
+
+			// One container per instance group
 			containers = append(containers, ct.newUtilContainer(ig.Name))
 		}
 	}
@@ -174,6 +181,13 @@ func (f *JobFactory) InstanceGroupManifestJob(manifest bdm.Manifest) (*qjv1a1.Qu
 	if err != nil {
 		return nil, err
 	}
+
+	// add BOSH link secret to the output list
+	for container, secret := range linkOutputs {
+		qJob.Spec.Output.OutputMap[container]["provides.json"] = qjv1a1.SecretOptions{
+			Name: secret,
+		}
+	}
 	return qJob, nil
 }
 
@@ -280,6 +294,11 @@ func (f *JobFactory) releaseImageQJob(name string, manifest bdm.Manifest, secret
 		}
 	}
 
+	outputMap := qjv1a1.OutputMap{}
+	for _, container := range containers {
+		outputMap[container.Name] = qjv1a1.NewFileToSecret("output.json", outputSecretNamePrefix+container.Name, true)
+	}
+
 	// Construct the "BPM configs" or "data gathering" auto-errand qJob
 	qJob := &qjv1a1.QuarksJob{
 		ObjectMeta: metav1.ObjectMeta{
@@ -291,12 +310,11 @@ func (f *JobFactory) releaseImageQJob(name string, manifest bdm.Manifest, secret
 		},
 		Spec: qjv1a1.QuarksJobSpec{
 			Output: &qjv1a1.Output{
-				NamePrefix: outputSecretNamePrefix,
+				OutputMap: outputMap,
 				SecretLabels: map[string]string{
 					bdv1.LabelDeploymentName:       manifest.Name,
 					bdv1.LabelDeploymentSecretType: secretType.String(),
 				},
-				Versioned: true,
 			},
 			Trigger: qjv1a1.Trigger{
 				Strategy: qjv1a1.TriggerOnce,