feat: Add ACI Spot virtual nodes support to virtual kubelet

.circleci/config.yml

@@ -1,6 +1,7 @@
 version: 2
 jobs:
   validate:
+    parallelism: 2
     docker:
       - image: golang:1.16
     working_directory: /go/src/github.com/virtual-kubelet/azure-aci

client/aci/create.go

@@ -10,19 +10,26 @@ import (
 	"net/url"
 
 	"github.com/virtual-kubelet/azure-aci/client/api"
+	"github.com/virtual-kubelet/virtual-kubelet/log"
 )
 
 // CreateContainerGroup creates a new Azure Container Instance with the
 // provided properties.
 // From: https://docs.microsoft.com/en-us/rest/api/container-instances/containergroups/createorupdate
 func (c *Client) CreateContainerGroup(ctx context.Context, resourceGroup, containerGroupName string, containerGroup ContainerGroup) (*ContainerGroup, error) {
+
+	// create a new VersionProvider object
+	versionProvider := newVersionProvider(apiVersion)
+
 	urlParams := url.Values{
-		"api-version": []string{apiVersion},
+		// use versionProvider to get the correct min api version
+		"api-version": []string{versionProvider.getVersion(containerGroup, ctx).finalVersion},
 	}
 
 	// Create the url.
 	uri := api.ResolveRelative(c.auth.ResourceManagerEndpoint, containerGroupURLPath)
 	uri += "?" + url.Values(urlParams).Encode()
+	log.G(ctx).Infof("Using url '%s' for Create", uri)
 
 	// Create the body for the request.
 	b := new(bytes.Buffer)

client/aci/types.go

@@ -28,6 +28,16 @@ const (
 	OnFailure ContainerGroupRestartPolicy = "OnFailure"
 )
 
+// ContainerGroupPriority enumerates the values for container group priority.
+type ContainerGroupPriority string
+
+const (
+	// Regular specifies the regular priority for container group restart priority.
+	Regular ContainerGroupPriority = "Regular"
+	// Spot specifies the spot priority for container group restart priority.
+	Spot ContainerGroupPriority = "Spot"
+)
+
 // ContainerNetworkProtocol enumerates the values for container network protocol.
 type ContainerNetworkProtocol string
 
@@ -94,9 +104,10 @@ type ContainerGroupProperties struct {
 	Volumes                  []Volume                             `json:"volumes,omitempty"`
 	InstanceView             ContainerGroupPropertiesInstanceView `json:"instanceView,omitempty"`
 	Diagnostics              *ContainerGroupDiagnostics           `json:"diagnostics,omitempty"`
-	SubnetIds           	 []*SubnetIdDefinition            `json:"subnetIds,omitempty"`
+	SubnetIds                []*SubnetIdDefinition                `json:"subnetIds,omitempty"`
 	Extensions               []*Extension                         `json:"extensions,omitempty"`
 	DNSConfig                *DNSConfig                           `json:"dnsConfig,omitempty"`
+	Priority                 ContainerGroupPriority               `json:"priority,omitempty"`
 }
 
 // ContainerGroupPropertiesInstanceView is the instance view of the container group. Only valid in response.
@@ -105,7 +116,7 @@ type ContainerGroupPropertiesInstanceView struct {
 	State  string  `json:"state,omitempty"`
 }
 
-// SubnetIdDefinition is the subnet ID, the format should be 
+// SubnetIdDefinition is the subnet ID, the format should be
 // /subscriptions/{subscriptionID}/resourceGroups/{ResourceGroup}/providers/Microsoft.Network/virtualNetworks/{VNET}/subnets/{Subnet}
 type SubnetIdDefinition struct {
 	ID string `json:"id,omitempty"`
@@ -242,7 +253,7 @@ type GPUSKU string
 
 const (
 	// K80 specifies the K80 GPU SKU
-	K80  GPUSKU = "K80"
+	K80 GPUSKU = "K80"
 	// P100 specifies the P100 GPU SKU
 	P100 GPUSKU = "P100"
 	// V100 specifies the V100 GPU SKU
@@ -460,7 +471,7 @@ type ExtensionType string
 
 // Supported extension types
 const (
-	ExtensionTypeKubeProxy ExtensionType = "kube-proxy"
+	ExtensionTypeKubeProxy       ExtensionType = "kube-proxy"
 	ExtensionTypeRealtimeMetrics ExtensionType = "realtime-metrics"
 )
 

client/aci/versioning.go

@@ -0,0 +1,70 @@
+package aci
+
+import (
+    "github.com/virtual-kubelet/virtual-kubelet/log"
+    "context"
+)
+
+//<summary>
+//	map of minimum api version required for different properties
+//	TODO: read from a separate json file if it gets too large
+//</summary>
+var minVersionSupport = map[string]string {
+    "Priority": "2021-10-01",
+}
+
+//<summary>
+//	VersionProvider struct for selecting api version based on various properties
+//	Maintains the api version selected after checking certain property
+//<summary>
+type VersionProvider struct {
+    finalVersion string
+}
+
+//<summary>
+//	creates a new instance of VErsionProvider, and sets default version
+//	assumes api version to be of the format YYYY-mm-dd[-suffix]
+//	assumes that the api version format will not be violated
+//</summary>
+//<param name="defaultVersion"> The default api version </param>
+//<returns>
+//	reference to  an instance of the verison provider object
+//</returns>
+func newVersionProvider(defaultVersion string) (*VersionProvider) {
+    return &VersionProvider{defaultVersion}
+}
+
+//<summary>
+//	get the api version for the specific ContainerGroup instance based on various properties
+//</summary>
+//<param name="containerGroup"> the ContainerGroup instance for which version is to be selected </param>
+//<param name="ctx"> the Context to be used for logging </param>
+//<returns>
+//	reference to an instance of VersionProvider with the finalVersion field updated
+//</returns>
+func (versionProvider *VersionProvider) getVersion(containerGroup ContainerGroup, ctx context.Context) (* VersionProvider) {
+
+    versionProvider.setVersionFromProperty(string(containerGroup.ContainerGroupProperties.Priority), "Priority", ctx)
+
+    log.G(ctx).Infof("API Version set to %s \n", versionProvider.finalVersion)
+    return versionProvider
+}
+
+//<summary>
+//	find the min api version for a string property based on the value in minVersionSupport map
+//	assumes that the api version always uses the correct format YYYY-mm-dd[-suffix]
+//</summary>
+//<param name="property">the string value of some property field that should be checked<param>
+//<param name="keyRef">the key for the property in the minVersionSupport map</param>
+//<param name="ctx">the context to be used for logging</param>
+//<returns>
+//	reference to an instance of VersionProvider with the finalVersion field updated
+//</returns>
+func (versionProvider *VersionProvider) setVersionFromProperty(property string, keyRef string, ctx context.Context) (*VersionProvider) {
+    minVersion, ok := minVersionSupport[keyRef]
+    if len(property) > 0 && ok && versionProvider.finalVersion < minVersion {
+        versionProvider.finalVersion = minVersion
+    }
+    log.G(ctx).Infof("Selected API Version %s for property %s with value %s \n", versionProvider.finalVersion, keyRef, property)
+    return versionProvider
+}

client/aci/versioning_test.go

@@ -0,0 +1,85 @@
+package aci
+
+import (
+	"testing"
+	"context"
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
+)
+
+// tests setlecting version when property is present in map
+func TestSetVersionFromPropertyInMap(t *testing.T) {
+	key := "Priority"
+	defaultVersion := apiVersion
+	versionProvider:= newVersionProvider(defaultVersion)
+
+	// should use defualt version when property is not populated
+	versionProvider.setVersionFromProperty("", key, context.Background())
+	assert.Check(t, is.Equal(versionProvider.finalVersion, defaultVersion), "Default version should be used when a property value is empty")
+
+	// should use api version present in map if priority is Spot
+	versionProvider.setVersionFromProperty("Spot", key, context.Background())
+	assert.Check(t, is.Equal(versionProvider.finalVersion, minVersionSupport[key]), "When a version is available for the property in map, the final version should be >= min api version for the property")
+
+	// should use api version present in map if priority is Regular
+	versionProvider.setVersionFromProperty("Regular", key, context.Background())
+	assert.Check(t, is.Equal(versionProvider.finalVersion, minVersionSupport[key]), "When a version is available for the property in map, the final version should be >= min api version for the property")
+
+}
+
+// tests selecting version when a property is present in map with version < defualtVersion
+func TestSetLowerVersionFromPropertyInMap (t *testing.T) {
+	key := "Priority"
+	largeDefaultVersion := "9999-99-99"
+	versionProvider := newVersionProvider(largeDefaultVersion)
+
+	// should use largeDefaultVersion as it is > the min api version for this key
+	versionProvider.setVersionFromProperty("Regular", key, context.Background())
+	assert.Check(t, versionProvider.finalVersion >= minVersionSupport[key], "Use larger version among default and min api versions for various properties")
+}
+
+// test selecting version when property is not present in map
+func TestSetVersionFromPropertyNotInMap(t *testing.T) {
+	key := "someUnknownKey"
+	defaultVersion := apiVersion
+	versionProvider:= newVersionProvider(defaultVersion)
+
+	// should use default version when property is not present in map
+	versionProvider.setVersionFromProperty("propertyValue", key, context.Background())
+	assert.Check(t, versionProvider.finalVersion == defaultVersion, "Default version should be used when no version for a property is available in map")
+}
+
+// test getVersion for ContainerGroup with Priority
+func TestGetVersionForContainerGroupWithPriority(t *testing.T) {
+	key := "Priority"
+	versionProvider := newVersionProvider(apiVersion)
+	containerGroup := ContainerGroup{
+		Location: "eastus",
+		ContainerGroupProperties: ContainerGroupProperties{
+			Priority: Spot,
+			OsType: Linux,
+		},
+	}
+
+	// should use api version in map when priority is set for a containerGroup
+	versionProvider.getVersion(containerGroup, context.Background())
+	assert.Check(t, is.Equal(versionProvider.finalVersion, minVersionSupport[key]), "Use api version in map when priority is set for a containerGroup")
+
+}
+
+// test getVersion for containerGroup without Priority
+func TestGetVersionForContainerGroupWithoutPriority(t *testing.T) {
+	defaultVersion := apiVersion
+	versionProvider := newVersionProvider(defaultVersion)
+	containerGroup := ContainerGroup{
+		Location: "eastus",
+		ContainerGroupProperties: ContainerGroupProperties{
+			OsType: Linux,
+		},
+	}
+
+	// should use default api version when priority is not set for a containerGroup
+	versionProvider.getVersion(containerGroup, context.Background())
+	assert.Check(t, is.Equal(versionProvider.finalVersion, defaultVersion), "Use default api version when priority is not set for a containerGroup")
+
+}

provider/aci.go

@@ -67,6 +67,11 @@ const (
 	gpuTypeAnnotation = "virtual-kubelet.io/gpu-type"
 )
 
+const (
+	priorityTypeAnnotation = "virtual-kubelet.io/priority"
+	priorityTagName	       = "virtual-kubelet.io-priority"
+)
+
 const (
 	statusReasonPodDeleted            = "NotFound"
 	statusMessagePodDeleted           = "The pod may have been deleted from the provider"
@@ -724,6 +729,11 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error {
 		"CreationTimestamp": podCreationTimestamp,
 	}
 
+	//set container group priority property and Tag
+	if err := setContainerGroupPriority(&containerGroup, pod); err != nil {
+		return fmt.Errorf("error setting container group priority: %v", err)
+	}
+
 	p.amendVnetResources(&containerGroup, pod)
 	if p.realtimeMetricsExtension != nil {
 		containerGroup.ContainerGroupProperties.Extensions = append(containerGroup.ContainerGroupProperties.Extensions, p.realtimeMetricsExtension)
@@ -734,6 +744,30 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error {
 	return p.createContainerGroup(ctx, pod.Namespace, pod.Name, &containerGroup)
 }
 
+// Set the Container Group Priority Property and Tag
+// Set the Container Group Priority Tag with Tag name virtual-kubelet.io-priority
+// value is set based on the priorityTypeAnnotation field under annotations in the pod spec
+// Accepted Values : Regular, Spot
+func setContainerGroupPriority(containerGroup *aci.ContainerGroup, pod *v1.Pod) error {
+
+	if pod.Annotations != nil {
+		priority, priorityExists := pod.Annotations[priorityTypeAnnotation]
+		if priorityExists {
+			if strings.EqualFold(priority, string(aci.Spot)) {
+				containerGroup.ContainerGroupProperties.Priority = aci.Spot
+				containerGroup.Tags[priorityTagName] = string(aci.Spot)
+			} else if strings.EqualFold(priority, string(aci.Regular)) {
+				containerGroup.ContainerGroupProperties.Priority = aci.Regular
+				containerGroup.Tags[priorityTagName] = string(aci.Regular)
+			} else {
+				return fmt.Errorf("The pod requires either Regular or Spot priority. Invalid value %s", priority)
+			}
+		}
+	}
+
+	return nil
+}
+
 func (p *ACIProvider) createContainerGroup(ctx context.Context, podNS, podName string, cg *aci.ContainerGroup) error {
 	ctx, span := trace.StartSpan(ctx, "aci.createContainerGroup")
 	defer span.End()