Merge pull request #739 from JoshVanL/415-default-amis

Fallback to pre-built AMI images if none privately built

pkg/apis/cluster/v1alpha1/constants.go

@@ -6,4 +6,7 @@ const (
 	KubernetesMasterRoleName = "master"
 	KubernetesWorkerRoleName = "worker"
 	KubernetesEtcdRoleName   = "etcd"
+
+	ImageBaseDefault       = "centos-puppet-agent"
+	ImageBaseDefaultWorker = "centos-puppet-agent-k8s-worker"
 )

pkg/apis/cluster/v1alpha1/defaults.go

@@ -195,9 +195,9 @@ func SetDefaults_InstancePool(obj *InstancePool) {
 	// set image to default image
 	if obj.Image == "" {
 		if obj.Type == InstancePoolTypeWorker {
-			obj.Image = "centos-puppet-agent-k8s-worker"
+			obj.Image = ImageBaseDefaultWorker
 		} else {
-			obj.Image = "centos-puppet-agent"
+			obj.Image = ImageBaseDefault
 		}
 	}
 

pkg/packer/packer.go

@@ -10,6 +10,7 @@ import (
 
 	tarmakv1alpha1 "github.com/jetstack/tarmak/pkg/apis/tarmak/v1alpha1"
 	"github.com/jetstack/tarmak/pkg/tarmak/interfaces"
+	"github.com/jetstack/tarmak/pkg/version"
 )
 
 type Packer struct {
@@ -33,7 +34,7 @@ func New(tarmak interfaces.Tarmak) *Packer {
 }
 
 // List existing images
-func (p *Packer) List() ([]tarmakv1alpha1.Image, error) {
+func (p *Packer) List() ([]*tarmakv1alpha1.Image, error) {
 	return p.tarmak.Cluster().Environment().Provider().QueryImages(
 		map[string]string{tarmakv1alpha1.ImageTagEnvironment: p.tarmak.Environment().Name()},
 	)
@@ -116,5 +117,21 @@ func (p *Packer) IDs(encrypted bool) (map[string]string, error) {
 		}
 	}
 
+	if len(imageIDByName) == 0 && !encrypted {
+		version := version.CleanVersion()
+		p.log.Warn("no built images found")
+
+		image, err := p.tarmak.Cluster().Environment().Provider().DefaultImage(version)
+		if err != nil {
+			return nil, err
+		}
+
+		imagesChangeTime[image.BaseImage] = image.CreationTimestamp.Time
+		imageIDByName[image.BaseImage] = image.Name
+
+		p.log.Warnf("EBS is unencrypted so using Jetstack's pre-built default image from version %s",
+			version)
+	}
+
 	return imageIDByName, nil
 }

pkg/tarmak/interfaces/interfaces.go

@@ -122,7 +122,8 @@ type Provider interface {
 	PublicZone() string
 	Environment() ([]string, error)
 	Variables() map[string]interface{}
-	QueryImages(tags map[string]string) ([]tarmakv1alpha1.Image, error)
+	QueryImages(tags map[string]string) ([]*tarmakv1alpha1.Image, error)
+	DefaultImage(version string) (*tarmakv1alpha1.Image, error)
 	VaultKV() (kv.Service, error)
 	VaultKVWithParams(kmsKeyID, unsealKeyName string) (kv.Service, error)
 	ListHosts(Cluster) ([]Host, error)
@@ -203,7 +204,7 @@ type Config interface {
 
 type Packer interface {
 	IDs(encrypted bool) (map[string]string, error)
-	List() ([]tarmakv1alpha1.Image, error)
+	List() ([]*tarmakv1alpha1.Image, error)
 	Build(imageNames []string) error
 }
 

pkg/tarmak/provider/amazon/image.go

@@ -7,17 +7,61 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1"
 	tarmakv1alpha1 "github.com/jetstack/tarmak/pkg/apis/tarmak/v1alpha1"
 )
 
-func (a *Amazon) QueryImages(tags map[string]string) (images []tarmakv1alpha1.Image, err error) {
+const (
+	defaultImagesOwner = "344758251446"
+)
 
+func (a *Amazon) DefaultImage(version string) (*tarmakv1alpha1.Image, error) {
 	sess, err := a.Session()
 	if err != nil {
-		return images, err
+		return nil, err
+	}
+	svc := ec2.New(sess)
+
+	name := aws.String(fmt.Sprintf("Tarmak %s*", version))
+	amis, err := svc.DescribeImages(&ec2.DescribeImagesInput{
+		Owners: []*string{
+			aws.String(defaultImagesOwner),
+		},
+		Filters: []*ec2.Filter{
+			&ec2.Filter{
+				Name:   aws.String("name"),
+				Values: []*string{name},
+			},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if amis == nil || len(amis.Images) == 0 {
+		return nil, fmt.Errorf("failed to find pre-made AMI image with name: %s", *name)
+	}
+
+	image, err := a.setImageTags(amis.Images[0])
+	if err != nil {
+		return nil, err
+	}
+
+	if image.BaseImage == "" {
+		image.BaseImage = clusterv1alpha1.ImageBaseDefault
 	}
 
+	return image, nil
+}
+
+func (a *Amazon) QueryImages(tags map[string]string) (images []*tarmakv1alpha1.Image, err error) {
+
+	sess, err := a.Session()
+	if err != nil {
+		return images, err
+	}
 	svc := ec2.New(sess)
 
 	filters := []*ec2.Filter{}
@@ -35,56 +79,70 @@ func (a *Amazon) QueryImages(tags map[string]string) (images []tarmakv1alpha1.Im
 		return images, err
 	}
 
-	formatRFC3339amazon := "2006-01-02T15:04:05.999Z07:00"
-
 	for _, ami := range amis.Images {
-		image := tarmakv1alpha1.Image{}
-		image.Annotations = map[string]string{}
-
-		// copy over tags from the AMI to image annotations
-		for _, tag := range ami.Tags {
-			image.Annotations[*tag.Key] = *tag.Value
-			// copy over base image name from AMI tags
-			if *tag.Key == tarmakv1alpha1.ImageTagBaseImageName {
-				image.BaseImage = *tag.Value
-			}
+		image, err := a.setImageTags(ami)
+		if err != nil {
+			return nil, err
 		}
 
-		creationTimestamp, err := time.Parse(formatRFC3339amazon, *ami.CreationDate)
-		if err != nil {
-			return images, fmt.Errorf("error parsing time stamp '%s'", err)
+		if image != nil {
+			images = append(images, image)
 		}
-		image.CreationTimestamp.Time = creationTimestamp
-		image.Name = *ami.ImageId
-		image.Location = a.Region()
+	}
+
+	return images, nil
+}
+
+func (a *Amazon) setImageTags(ami *ec2.Image) (*tarmakv1alpha1.Image, error) {
+	image := &tarmakv1alpha1.Image{
+		ObjectMeta: metav1.ObjectMeta{
+			Annotations: make(map[string]string),
+		},
+	}
 
-		if ami.RootDeviceName == nil {
-			a.log.Warnf("failed to obtain root device name of ami '%s'", image.Name)
-			continue
+	// copy over tags from the AMI to image annotations
+	for _, tag := range ami.Tags {
+		image.Annotations[*tag.Key] = *tag.Value
+		// copy over base image name from AMI tags
+		if *tag.Key == tarmakv1alpha1.ImageTagBaseImageName {
+			image.BaseImage = *tag.Value
 		}
-		rootName := *ami.RootDeviceName
-
-		foundRoot := false
-		for _, d := range ami.BlockDeviceMappings {
-			if d.DeviceName != nil && *d.DeviceName == rootName {
-				if d.Ebs == nil || d.Ebs.Encrypted == nil {
-					a.log.Warnf("failed to determine the encryption state of ami '%s'", image.Name)
-					continue
-				}
-
-				image.Encrypted = *d.Ebs.Encrypted
-				foundRoot = true
-				break
+	}
+
+	formatRFC3339amazon := "2006-01-02T15:04:05.999Z07:00"
+	creationTimestamp, err := time.Parse(formatRFC3339amazon, *ami.CreationDate)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing time stamp '%s'", err)
+	}
+
+	image.CreationTimestamp.Time = creationTimestamp
+	image.Name = *ami.ImageId
+	image.Location = a.Region()
+
+	if ami.RootDeviceName == nil {
+		a.log.Warnf("failed to obtain root device name of ami '%s'", image.Name)
+		return nil, nil
+	}
+	rootName := *ami.RootDeviceName
+
+	foundRoot := false
+	for _, d := range ami.BlockDeviceMappings {
+		if d.DeviceName != nil && *d.DeviceName == rootName {
+			if d.Ebs == nil || d.Ebs.Encrypted == nil {
+				a.log.Warnf("failed to determine the encryption state of ami '%s'", image.Name)
+				continue
 			}
-		}
 
-		if !foundRoot {
-			a.log.Warnf("failed to find root device of ami '%s'", image.Name)
-			continue
+			image.Encrypted = *d.Ebs.Encrypted
+			foundRoot = true
+			break
 		}
+	}
 
-		images = append(images, image)
+	if !foundRoot {
+		a.log.Warnf("failed to find root device of ami '%s'", image.Name)
+		return nil, nil
 	}
 
-	return images, nil
+	return image, nil
 }

pkg/tarmak/tarmak_test.go

@@ -49,7 +49,7 @@ func (tt *testTarmak) finish() {
 }
 
 func (tt *testTarmak) fakeAWSProvider(name string) {
-	baseImage := tarmakv1alpha1.Image{}
+	baseImage := &tarmakv1alpha1.Image{}
 	baseImage.Name = "ami-6e28b517"
 
 	tt.fakeProvider.EXPECT().Name().AnyTimes().Return(name)
@@ -59,7 +59,7 @@ func (tt *testTarmak) fakeAWSProvider(name string) {
 	tt.fakeProvider.EXPECT().Validate().AnyTimes().Return(nil)
 	tt.fakeProvider.EXPECT().RemoteState(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return("\n")
 	tt.fakeProvider.EXPECT().RemoteStateBucketName().AnyTimes().Return("my-remote-bucket")
-	tt.fakeProvider.EXPECT().QueryImages(gomock.Any()).AnyTimes().Return([]tarmakv1alpha1.Image{baseImage}, nil)
+	tt.fakeProvider.EXPECT().QueryImages(gomock.Any()).AnyTimes().Return([]*tarmakv1alpha1.Image{baseImage}, nil)
 	tt.fakeProvider.EXPECT().Variables().AnyTimes().Return(map[string]interface{}{
 		"test": "ffs",
 	})

pkg/version/version.go

@@ -20,6 +20,7 @@ package version
 import (
 	"fmt"
 	"runtime"
+	"strings"
 
 	apimachineryversion "k8s.io/apimachinery/pkg/version"
 )
@@ -41,3 +42,31 @@ func Get() apimachineryversion.Info {
 		Platform:     fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH),
 	}
 }
+
+func CleanVersion() string {
+	if !strings.HasSuffix(gitMinor, "+") {
+		return gitVersion
+	}
+
+	var out []string
+	split := strings.Split(gitVersion, "-")
+
+LOOP:
+	for _, s := range split {
+		for _, preRelease := range []string{"alpha", "beta"} {
+
+			if strings.Contains(s, preRelease) {
+				out = append(out, strings.Split(s, ".")[0])
+				break LOOP
+			}
+		}
+
+		if strings.HasPrefix(s, "dirty") {
+			break
+		}
+
+		out = append(out, s)
+	}
+
+	return strings.Join(out, "-")
+}

pkg/version/version_test.go

@@ -0,0 +1,56 @@
+// Copyright Jetstack Ltd. See LICENSE for details.
+package version
+
+import (
+	"testing"
+)
+
+func Test_CleanVersion(t *testing.T) {
+	for _, test := range []struct {
+		version, minor, exp string
+	}{
+		{
+			version: "0.6.0-alpha2.6+b364c17a97386f-dirty",
+			minor:   "6+",
+			exp:     "0.6.0-alpha2",
+		},
+
+		{
+			version: "0.6.0-alpha2-dirty",
+			minor:   "6+",
+			exp:     "0.6.0-alpha2",
+		},
+
+		{
+			version: "0.6.0-alpha2",
+			minor:   "6+",
+			exp:     "0.6.0-alpha2",
+		},
+
+		{
+			version: "0.6.0",
+			minor:   "6",
+			exp:     "0.6.0",
+		},
+
+		{
+			version: "0.6.1",
+			minor:   "6",
+			exp:     "0.6.1",
+		},
+
+		{
+			version: "0.6.0-dirty",
+			minor:   "6+",
+			exp:     "0.6.0",
+		},
+	} {
+
+		gitVersion = test.version
+		gitMinor = test.minor
+
+		if got := CleanVersion(); got != test.exp {
+			t.Errorf("clear version failed, exp=%s\ngot=%s", test, got)
+		}
+	}
+}