Add YAML model structs for manifest files (#194)

Part of #191. These structs will represent the contents of the new "manifest" file, which contains all the information we'll need in the future to cleanly upgrade a template output. This PR contains custom unmarshaling but not custom marshaling. That will be a separate PR. Example manifest file: ``` api_version: 'cli.abcxyz.dev/v1alpha1' template_location: 'github.com/abcxyz/abc.git//t/rest_server' template_dirhash: 'h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03' inputs: - name: 'my_input_1' value: 'my_value_1' - name: 'my_input_2' value: 'my_value_2' output_hashes: - file: 'a/b/c.txt' hash: 'h1:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c' - file: 'd/e/f.txt' hash: 'h1:7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730' ``` Some of the field names have been changed since the design doc because the old names seemed unclear.
abcxyz · Sep 8, 2023 · de35382 · de35382
1 parent 6eb9bad
commit de35382
Show file tree

Hide file tree

Showing 9 changed files with 400 additions and 31 deletions.
diff --git a/templates/commands/render/render_test.go b/templates/commands/render/render_test.go
@@ -307,7 +307,7 @@ steps:
 			wantTemplateContents: map[string]string{
 				"spec.yaml": "this is an unparseable YAML file *&^#%$",
 			},
-			wantErr: "error parsing YAML spec file",
+			wantErr: "error parsing spec",
 		},
 		{
 			name: "existing_dest_file_with_overwrite_flag_should_succeed",

diff --git a/templates/model/manifest/manifest.go b/templates/model/manifest/manifest.go
@@ -0,0 +1,127 @@
+// Copyright 2023 The Authors (see AUTHORS file)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package manifest
+
+import (
+	"errors"
+	"io"
+
+	"github.com/abcxyz/abc/templates/model"
+	"gopkg.in/yaml.v3"
+)
+
+// Decode unmarshals the YAML Manifest from r. This function exists so we can
+// validate the Manifest model before providing it to the caller; we don't want the
+// caller to forget, and thereby introduce bugs.
+//
+// If the Manifest parses successfully but then fails validation, the manifest will be
+// returned along with the validation error.
+func Decode(r io.Reader) (*Manifest, error) {
+	out := &Manifest{}
+	if err := model.DecodeAndValidate(r, "manifest", out); err != nil {
+		return out, err //nolint:wrapcheck
+	}
+	return out, nil
+}
+
+// Manifest represents the contents of a manifest file. A manifest file is the
+// set of all information that is needed to cleanly upgrade to a new template
+// version in the future.
+type Manifest struct {
+	Pos model.ConfigPos `yaml:"-"`
+
+	APIVersion model.String `yaml:"api_version"`
+
+	// The template source address as passed to `abc templates render`.
+	TemplateLocation model.String `yaml:"template_location"`
+
+	// The dirhash (https://pkg.go.dev/golang.org/x/mod/sumdb/dirhash) of the
+	// template source tree (not the output). This shows exactly what version of
+	// the template was installed.
+	TemplateDirhash model.String `yaml:"template_dirhash"`
+
+	// The input values that were supplied by the user when rendering the template.
+	Inputs []*Input `yaml:"inputs"`
+
+	// The hash of each output file created by the template.
+	OutputHashes []*OutputHash `yaml:"output_hashes"`
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler.
+func (m *Manifest) UnmarshalYAML(n *yaml.Node) error {
+	return model.UnmarshalPlain(n, m, &m.Pos) //nolint:wrapcheck
+}
+
+// Validate() implements model.Validator.
+func (m *Manifest) Validate() error {
+	// Inputs and OutputHashes can legally be empty, since a template doesn't
+	// necessarily have these.
+	return errors.Join(
+		model.IsKnownSchemaVersion(&m.Pos, m.APIVersion, "api_version"),
+		model.NotZeroModel(&m.Pos, m.TemplateLocation, "template_location"),
+		model.NotZeroModel(&m.Pos, m.TemplateDirhash, "template_dirhash"),
+		model.ValidateEach(m.Inputs),
+		model.ValidateEach(m.OutputHashes),
+	)
+}
+
+// Input is a YAML object representing an input value that was provided to the
+// template when it was rendered.
+type Input struct {
+	Pos model.ConfigPos
+
+	// The name of the template input, e.g. "my_service_account"
+	Name model.String `yaml:"name"`
+	// The value of the template input, e.g. "[email protected]".
+	Value model.String `yaml:"value"`
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler.
+func (i *Input) UnmarshalYAML(n *yaml.Node) error {
+	return model.UnmarshalPlain(n, i, &i.Pos) //nolint:wrapcheck
+}
+
+// Validate() implements model.Validator.
+func (i *Input) Validate() error {
+	return errors.Join(
+		model.NotZeroModel(&i.Pos, i.Name, "name"),
+		model.NotZeroModel(&i.Pos, i.Value, "value"),
+	)
+}
+
+// OutputHash records a checksum of a single file as it was created during
+// template rendering.
+type OutputHash struct {
+	Pos model.ConfigPos
+
+	// The path, relative to the destination directory, of this file.
+	File model.String `yaml:"file"`
+	// The dirhash-style hash (see https://pkg.go.dev/golang.org/x/mod/sumdb/dirhash)
+	// of this file. The format looks like "h1:0a1b2c3d...".
+	Hash model.String `yaml:"hash"`
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler.
+func (f *OutputHash) UnmarshalYAML(n *yaml.Node) error {
+	return model.UnmarshalPlain(n, f, &f.Pos) //nolint:wrapcheck
+}
+
+// Validate() implements model.Validator.
+func (f *OutputHash) Validate() error {
+	return errors.Join(
+		model.NotZeroModel(&f.Pos, f.File, "file"),
+		model.NotZeroModel(&f.Pos, f.Hash, "hash"),
+	)
+}
diff --git a/templates/model/manifest/manifest_test.go b/templates/model/manifest/manifest_test.go
@@ -0,0 +1,220 @@
+// Copyright 2023 The Authors (see AUTHORS file)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package manifest
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/abcxyz/abc/templates/model"
+	"github.com/abcxyz/pkg/testutil"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"gopkg.in/yaml.v3"
+)
+
+func TestDecode(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name             string
+		in               string
+		want             *Manifest
+		wantUnmarshalErr string
+		wantValidateErr  string
+	}{
+		{
+			name: "simple_success",
+			in: `
+api_version: 'cli.abcxyz.dev/v1alpha1'
+template_location: 'github.com/abcxyz/abc.git//t/rest_server'
+template_dirhash: 'h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03'
+inputs:
+  - name: 'my_input_1'
+    value: 'my_value_1'
+  - name: 'my_input_2'
+    value: 'my_value_2'
+output_hashes:
+  - file: 'a/b/c.txt'
+    hash: 'h1:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c'
+  - file: 'd/e/f.txt'
+    hash: 'h1:7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730'`,
+			want: &Manifest{
+				APIVersion:       model.String{Val: "cli.abcxyz.dev/v1alpha1"},
+				TemplateLocation: model.String{Val: "github.com/abcxyz/abc.git//t/rest_server"},
+				TemplateDirhash:  model.String{Val: "h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03"},
+				Inputs: []*Input{
+					{
+						Name:  model.String{Val: "my_input_1"},
+						Value: model.String{Val: "my_value_1"},
+					},
+					{
+						Name:  model.String{Val: "my_input_2"},
+						Value: model.String{Val: "my_value_2"},
+					},
+				},
+				OutputHashes: []*OutputHash{
+					{
+						File: model.String{Val: "a/b/c.txt"},
+						Hash: model.String{Val: "h1:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
+					},
+					{
+						File: model.String{Val: "d/e/f.txt"},
+						Hash: model.String{Val: "h1:7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730"},
+					},
+				},
+			},
+		},
+		{
+			name: "fields_missing",
+			in:   `api_version: "foo"`,
+			wantValidateErr: `at line 1 column 14: field "api_version" value must be one of [cli.abcxyz.dev/v1alpha1]
+at line 1 column 1: field "template_location" is required
+at line 1 column 1: field "template_dirhash" is required`,
+		},
+		{
+			name: "input_missing_name",
+			in: `
+api_version: 'cli.abcxyz.dev/v1alpha1'
+template_location: 'github.com/abcxyz/abc.git//t/rest_server'
+template_dirhash: 'h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03'
+inputs:
+  - value: 'my_value_1'
+output_hashes:
+  - file: 'a/b/c.txt'
+    hash: 'h1:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c'`,
+			wantValidateErr: `at line 6 column 5: field "name" is required`,
+		},
+		{
+			name: "input_missing_value",
+			in: `
+api_version: 'cli.abcxyz.dev/v1alpha1'
+template_location: 'github.com/abcxyz/abc.git//t/rest_server'
+template_dirhash: 'h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03'
+inputs:
+  - name: 'my_input_1'
+output_hashes:
+  - file: 'a/b/c.txt'
+    hash: 'h1:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c'`,
+			wantValidateErr: `at line 6 column 5: field "value" is required`,
+		},
+		{
+			name: "output_hash_missing_file",
+			in: `
+api_version: 'cli.abcxyz.dev/v1alpha1'
+template_location: 'github.com/abcxyz/abc.git//t/rest_server'
+template_dirhash: 'h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03'
+inputs:
+  - name: 'my_input_1'
+    value: 'my_value_1'
+output_hashes:
+  - hash: 'h1:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c'`,
+			wantValidateErr: `at line 9 column 5: field "file" is required`,
+		},
+		{
+			name: "output_hash_missing_file",
+			in: `
+api_version: 'cli.abcxyz.dev/v1alpha1'
+template_location: 'github.com/abcxyz/abc.git//t/rest_server'
+template_dirhash: 'h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03'
+inputs:
+  - name: 'my_input_1'
+    value: 'my_value_1'
+output_hashes:
+  - file: 'a/b/c.txt'`,
+			wantValidateErr: `at line 9 column 5: field "hash" is required`,
+		},
+		{
+			name: "no_hashes", // It's rare but legal for a template to have no output files
+			in: `
+api_version: 'cli.abcxyz.dev/v1alpha1'
+template_location: 'github.com/abcxyz/abc.git//t/rest_server'
+template_dirhash: 'h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03'
+inputs:
+  - name: 'my_input_1'
+    value: 'my_value_1'
+`,
+			want: &Manifest{
+				APIVersion:       model.String{Val: "cli.abcxyz.dev/v1alpha1"},
+				TemplateLocation: model.String{Val: "github.com/abcxyz/abc.git//t/rest_server"},
+				TemplateDirhash:  model.String{Val: "h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03"},
+				Inputs: []*Input{
+					{
+						Name:  model.String{Val: "my_input_1"},
+						Value: model.String{Val: "my_value_1"},
+					},
+				},
+			},
+		},
+		{
+			name: "no_inputs", // It's legal for a template to have no inputs
+			in: `
+api_version: 'cli.abcxyz.dev/v1alpha1'
+template_location: 'github.com/abcxyz/abc.git//t/rest_server'
+template_dirhash: 'h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03'
+output_hashes:
+  - file: 'a/b/c.txt'
+    hash: 'h1:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c'`,
+			want: &Manifest{
+				APIVersion:       model.String{Val: "cli.abcxyz.dev/v1alpha1"},
+				TemplateLocation: model.String{Val: "github.com/abcxyz/abc.git//t/rest_server"},
+				TemplateDirhash:  model.String{Val: "h1:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03"},
+				OutputHashes: []*OutputHash{
+					{
+						File: model.String{Val: "a/b/c.txt"},
+						Hash: model.String{Val: "h1:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
+					},
+				},
+			},
+		},
+		{
+			name:             "bad_yaml_syntax",
+			in:               `[[[[[[[`,
+			wantUnmarshalErr: "did not find expected node content",
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := &Manifest{}
+			dec := yaml.NewDecoder(strings.NewReader(tc.in))
+			err := dec.Decode(got)
+
+			if diff := testutil.DiffErrString(err, tc.wantUnmarshalErr); diff != "" {
+				t.Fatal(diff)
+			}
+			if err != nil {
+				return
+			}
+
+			err = got.Validate()
+			if diff := testutil.DiffErrString(err, tc.wantValidateErr); diff != "" {
+				t.Fatal(diff)
+			}
+			if err != nil {
+				return
+			}
+
+			opt := cmpopts.IgnoreTypes(&model.ConfigPos{}, model.ConfigPos{}) // don't force test authors to assert the line and column numbers
+			if diff := cmp.Diff(got, tc.want, opt); diff != "" {
+				t.Errorf("unmarshaling didn't yield expected struct. Diff (-got +want): %s", diff)
+			}
+		})
+	}
+}
diff --git a/templates/model/pos.go b/templates/model/pos.go
@@ -33,6 +33,10 @@ type ConfigPos struct {
 	Column int
 }
 
+func (c ConfigPos) IsZero() bool {
+	return c == ConfigPos{}
+}
+
 // YAMLPos constructs a position struct based on a YAML parse cursor.
 func YAMLPos(n *yaml.Node) *ConfigPos {
 	return &ConfigPos{
@@ -51,7 +55,7 @@ func YAMLPos(n *yaml.Node) *ConfigPos {
 //	Creating an error: c.Errorf("something went wrong doing action %s", action)
 func (c *ConfigPos) Errorf(fmtStr string, args ...any) error {
 	err := fmt.Errorf(fmtStr, args...)
-	if c == nil || *c == (ConfigPos{}) {
+	if c == nil || c.IsZero() {
 		return err
 	}