Skip to content

Commit

Permalink
some refactoring after review
Browse files Browse the repository at this point in the history
  • Loading branch information
@r2k1
r2k1 committed Nov 15, 2024
1 parent 10bf050 commit 6fab59f
Show file tree
Hide file tree
Showing 5 changed files with 100 additions and 100 deletions.
4 changes: 2 additions & 2 deletions e2e/config/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -413,10 +413,10 @@ func (a *AzureClient) ensureReplication(ctx context.Context, image *Image, versi
if replicatedToCurrentRegion(version) {
return nil
}
return a.replicateToCurrentRegion(ctx, image, version)
return a.replicateImageVersionToCurrentRegion(ctx, image, version)
}

func (a *AzureClient) replicateToCurrentRegion(ctx context.Context, image *Image, version *armcompute.GalleryImageVersion) error {
func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, image *Image, version *armcompute.GalleryImageVersion) error {
galleryImageVersion, err := armcompute.NewGalleryImageVersionsClient(image.Gallery.SubscriptionID, a.Credential, a.ArmOptions)
if err != nil {
return fmt.Errorf("create a new images client: %v", err)
Expand Down
45 changes: 24 additions & 21 deletions e2e/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,26 +25,29 @@ var (
)

type Configuration struct {
AirgapNSGName string `env:"AIRGAP_NSG_NAME" envDefault:"abe2e-airgap-securityGroup"`
BlobContainer string `env:"BLOB_CONTAINER" envDefault:"abe2e"`
BlobStorageAccountPrefix string `env:"BLOB_STORAGE_ACCOUNT_PREFIX" envDefault:"abe2e"`
BuildID string `env:"BUILD_ID" envDefault:"local"`
DefaultSubnetName string `env:"DEFAULT_SUBNET_NAME" envDefault:"aks-subnet"`
E2ELoggingDir string `env:"LOGGING_DIR" envDefault:"scenario-logs"`
EnableAKSNodeControllerTest bool `env:"ENABLE_AKS_NODE_CONTROLLER_TEST"`
GalleryName string `env:"GALLERY_NAME" envDefault:"PackerSigGalleryEastUS"`
GalleryResourceGroup string `env:"GALLERY_RESOURCE_GROUP" envDefault:"aksvhdtestbuildrg"`
GallerySubscriptionID string `env:"GALLERY_SUBSCRIPTION_ID" envDefault:"c4c3550e-a965-4993-a50c-628fd38cd3e1"`
IgnoreScenariosWithMissingVHD bool `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD"`
KeepVMSS bool `env:"KEEP_VMSS"`
Location string `env:"LOCATION" envDefault:"westus3"`
SIGVersionTagName string `env:"SIG_VERSION_TAG_NAME" envDefault:"branch"`
SIGVersionTagValue string `env:"SIG_VERSION_TAG_VALUE" envDefault:"refs/heads/dev"`
SkipTestsWithSKUCapacityIssue bool `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE"`
SubscriptionID string `env:"SUBSCRIPTION_ID" envDefault:"8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8"`
TagsToRun string `env:"TAGS_TO_RUN"`
TagsToSkip string `env:"TAGS_TO_SKIP"`
TestTimeout time.Duration `env:"TEST_TIMEOUT" envDefault:"35m"`
AirgapNSGName string `env:"AIRGAP_NSG_NAME" envDefault:"abe2e-airgap-securityGroup"`
BlobContainer string `env:"BLOB_CONTAINER" envDefault:"abe2e"`
BlobStorageAccountPrefix string `env:"BLOB_STORAGE_ACCOUNT_PREFIX" envDefault:"abe2e"`
BuildID string `env:"BUILD_ID" envDefault:"local"`
DefaultSubnetName string `env:"DEFAULT_SUBNET_NAME" envDefault:"aks-subnet"`
E2ELoggingDir string `env:"LOGGING_DIR" envDefault:"scenario-logs"`
EnableAKSNodeControllerTest bool `env:"ENABLE_AKS_NODE_CONTROLLER_TEST"`
GalleryNameLinux string `env:"GALLERY_NAME" envDefault:"PackerSigGalleryEastUS"`
GalleryNameWindows string `env:"GALLERY_NAME_WINDOWS" envDefault:"AKSWindows"`
GalleryResourceGroupNameLinux string `env:"GALLERY_RESOURCE_GROUP" envDefault:"aksvhdtestbuildrg"`
GalleryResourceGroupNameWindows string `env:"GALLERY_RESOURCE_GROUP_WINDOWS" envDefault:"AKS-Windows"`
GallerySubscriptionIDLinux string `env:"GALLERY_SUBSCRIPTION_ID" envDefault:"c4c3550e-a965-4993-a50c-628fd38cd3e1"`
GallerySubscriptionIDWindows string `env:"GALLERY_SUBSCRIPTION_ID_WINDOWS" envDefault:"4be8920b-2978-43d7-ab14-04d8549c1d05"`
IgnoreScenariosWithMissingVHD bool `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD"`
KeepVMSS bool `env:"KEEP_VMSS"`
Location string `env:"LOCATION" envDefault:"westus3"`
SIGVersionTagName string `env:"SIG_VERSION_TAG_NAME" envDefault:"branch"`
SIGVersionTagValue string `env:"SIG_VERSION_TAG_VALUE" envDefault:"refs/heads/dev"`
SkipTestsWithSKUCapacityIssue bool `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE"`
SubscriptionID string `env:"SUBSCRIPTION_ID" envDefault:"8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8"`
TagsToRun string `env:"TAGS_TO_RUN"`
TagsToSkip string `env:"TAGS_TO_SKIP"`
TestTimeout time.Duration `env:"TEST_TIMEOUT" envDefault:"35m"`
}

func (c *Configuration) BlobStorageAccount() string {
Expand All @@ -56,7 +59,7 @@ func (c *Configuration) BlobStorageAccountURL() string {
}

func (c *Configuration) GalleryResourceID() string {
return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/galleries/%s", c.GallerySubscriptionID, c.GalleryResourceGroup, c.GalleryName)
return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/galleries/%s", c.GallerySubscriptionIDLinux, c.GalleryResourceGroupNameLinux, c.GalleryNameLinux)
}

func (c *Configuration) String() string {
Expand Down
12 changes: 6 additions & 6 deletions e2e/config/vhd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@ const (

var (
linuxGallery = &Gallery{
SubscriptionID: Config.GallerySubscriptionID,
ResourceGroupName: Config.GalleryResourceGroup,
Name: Config.GalleryName,
SubscriptionID: Config.GallerySubscriptionIDLinux,
ResourceGroupName: Config.GalleryResourceGroupNameLinux,
Name: Config.GalleryNameLinux,
}

windowsGallery = &Gallery{
SubscriptionID: "4be8920b-2978-43d7-ab14-04d8549c1d05",
ResourceGroupName: "AKS-Windows",
Name: "AKSWindows",
SubscriptionID: Config.GallerySubscriptionIDWindows,
ResourceGroupName: Config.GalleryResourceGroupNameWindows,
Name: Config.GalleryNameWindows,
}
)

Expand Down
43 changes: 0 additions & 43 deletions e2e/exec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,49 +55,6 @@ func (r podExecResult) dumpStderr(t *testing.T) {
}
}

func extractLogsFromVM(ctx context.Context, s *Scenario) (map[string]string, error) {
privateIP, err := getVMPrivateIPAddress(ctx, s)
require.NoError(s.T, err)

commandList := map[string]string{
"cluster-provision": "cat /var/log/azure/cluster-provision.log",
"kubelet": "journalctl -u kubelet",
"cluster-provision-cse-output": "cat /var/log/azure/cluster-provision-cse-output.log",
"sysctl-out": "sysctl -a",
"aks-node-controller": "cat /var/log/azure/aks-node-controller.log",
}

podName, err := getHostNetworkDebugPodName(ctx, s.Runtime.Cluster.Kube, s.T)
if err != nil {
return nil, fmt.Errorf("unable to get debug pod name: %w", err)
}

var result = map[string]string{}
for file, sourceCmd := range commandList {
s.T.Logf("executing command on remote VM at %s: %q", privateIP, sourceCmd)

execResult, err := execOnVM(ctx, s.Runtime.Cluster.Kube, privateIP, podName, string(s.Runtime.SSHKeyPrivate), sourceCmd, false)
if err != nil {
s.T.Logf("error fetching logs for %s: %s", file, err)
return nil, err
}
if execResult.stdout != nil {
out := execResult.stdout.String()
if out != "" {
result[file+".stdout.txt"] = out
}

}
if execResult.stderr != nil {
out := execResult.stderr.String()
if out != "" {
result[file+".stderr.txt"] = out
}
}
}
return result, nil
}

type ClusterParams struct {
CACert []byte
BootstrapToken string
Expand Down
96 changes: 68 additions & 28 deletions e2e/vmss.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func createVMSS(ctx context.Context, s *Scenario) *armcompute.VirtualMachineScal
require.NoError(s.T, err)
}

model := getBaseVMSSModel(s.Runtime.VMSSName, string(s.Runtime.SSHKeyPublic), nodeBootstrapping.CustomData, nodeBootstrapping.CSE, cluster)
model := getBaseVMSSModel(s, nodeBootstrapping.CustomData, nodeBootstrapping.CSE)

isAzureCNI, err := cluster.IsAzureCNI()
require.NoError(s.T, err, "checking if cluster is using Azure CNI")
Expand All @@ -59,20 +59,6 @@ func createVMSS(ctx context.Context, s *Scenario) *armcompute.VirtualMachineScal
require.NoError(s.T, err)
}

if s.VHD.Windows() {
model.Identity = &armcompute.VirtualMachineScaleSetIdentity{
Type: to.Ptr(armcompute.ResourceIdentityTypeSystemAssignedUserAssigned),
UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{
config.Config.VMIdentityResourceID(): {},
},
}
model.Properties.VirtualMachineProfile.StorageProfile.OSDisk.OSType = to.Ptr(armcompute.OperatingSystemTypesWindows)
model.Properties.VirtualMachineProfile.OSProfile.LinuxConfiguration = nil
model.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.Publisher = to.Ptr("Microsoft.Compute")
model.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.Type = to.Ptr("CustomScriptExtension")
model.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.TypeHandlerVersion = to.Ptr("1.10")
}

s.PrepareVMSSModel(ctx, s.T, &model)

operation, err := config.Azure.VMSS.BeginCreateOrUpdate(
Expand Down Expand Up @@ -110,15 +96,57 @@ func cleanupVMSS(ctx context.Context, s *Scenario) {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Minute)
defer cancel()
defer deleteVMSS(ctx, s)
extractLogsFromVM(ctx, s)
}

func extractLogsFromVM(ctx context.Context, s *Scenario) {
if s.VHD.Windows() {
extractLogsFromWindowsVM(ctx, s)
return
extractLogsFromVMWindows(ctx, s)
} else {
extractLogsFromVMLinux(ctx, s)
}
}

logFiles, err := extractLogsFromVM(ctx, s)
func extractLogsFromVMLinux(ctx context.Context, s *Scenario) {
privateIP, err := getVMPrivateIPAddress(ctx, s)
require.NoError(s.T, err)

commandList := map[string]string{
"cluster-provision": "cat /var/log/azure/cluster-provision.log",
"kubelet": "journalctl -u kubelet",
"cluster-provision-cse-output": "cat /var/log/azure/cluster-provision-cse-output.log",
"sysctl-out": "sysctl -a",
"aks-node-controller": "cat /var/log/azure/aks-node-controller.log",
}

podName, err := getHostNetworkDebugPodName(ctx, s.Runtime.Cluster.Kube, s.T)
if err != nil {
require.NoError(s.T, err)
}

var logFiles = map[string]string{}
for file, sourceCmd := range commandList {
s.T.Logf("executing command on remote VM at %s: %q", privateIP, sourceCmd)

execResult, err := execOnVM(ctx, s.Runtime.Cluster.Kube, privateIP, podName, string(s.Runtime.SSHKeyPrivate), sourceCmd, false)
if err != nil {
s.T.Logf("error fetching logs for %s: %s", file, err)
continue
}
if execResult.stdout != nil {
out := execResult.stdout.String()
if out != "" {
logFiles[file+".stdout.txt"] = out
}

}
if execResult.stderr != nil {
out := execResult.stderr.String()
if out != "" {
logFiles[file+".stderr.txt"] = out
}
}
}
err = dumpFileMapToDir(s.T, logFiles)
require.NoError(s.T, err)
}
Expand All @@ -144,9 +172,9 @@ $CollectedLogs=(Get-ChildItem . -Filter "*_logs.zip" -File)[0].Name
.\azcopy.exe copy "C:\k\containerd.err.log" "$arg1/containerd.err.log"
`

// extractLogsFromWindowsVM runs a script on windows VM to collect logs and upload them to a blob storage
// extractLogsFromVMWindows runs a script on windows VM to collect logs and upload them to a blob storage
// it then lists the blobs in the container and prints the content of each blob
func extractLogsFromWindowsVM(ctx context.Context, s *Scenario) {
func extractLogsFromVMWindows(ctx context.Context, s *Scenario) {
if !s.T.Failed() {
s.T.Logf("skipping logs extraction from windows VM, as the test didn't fail")
return
Expand Down Expand Up @@ -397,7 +425,7 @@ func generateVMSSName(s *Scenario) string {
return generateVMSSNameLinux(s.T)
}

func getBaseVMSSModel(name, sshPublicKey, customData, cseCmd string, cluster *Cluster) armcompute.VirtualMachineScaleSet {
func getBaseVMSSModel(s *Scenario, customData, cseCmd string) armcompute.VirtualMachineScaleSet {
model := armcompute.VirtualMachineScaleSet{
Location: to.Ptr(config.Config.Location),
SKU: &armcompute.SKU{
Expand All @@ -411,15 +439,14 @@ func getBaseVMSSModel(name, sshPublicKey, customData, cseCmd string, cluster *Cl
},
VirtualMachineProfile: &armcompute.VirtualMachineScaleSetVMProfile{
OSProfile: &armcompute.VirtualMachineScaleSetOSProfile{
ComputerNamePrefix: to.Ptr(name),
ComputerNamePrefix: to.Ptr(s.Runtime.VMSSName),
AdminUsername: to.Ptr("azureuser"),
AdminPassword: to.Ptr("pwnedPassword123!"),
CustomData: &customData,
LinuxConfiguration: &armcompute.LinuxConfiguration{
SSH: &armcompute.SSHConfiguration{
PublicKeys: []*armcompute.SSHPublicKey{
{
KeyData: to.Ptr(sshPublicKey),
KeyData: to.Ptr(string(s.Runtime.SSHKeyPublic)),
Path: to.Ptr("/home/azureuser/.ssh/authorized_keys"),
},
},
Expand All @@ -436,13 +463,13 @@ func getBaseVMSSModel(name, sshPublicKey, customData, cseCmd string, cluster *Cl
NetworkProfile: &armcompute.VirtualMachineScaleSetNetworkProfile{
NetworkInterfaceConfigurations: []*armcompute.VirtualMachineScaleSetNetworkConfiguration{
{
Name: to.Ptr(name),
Name: to.Ptr(s.Runtime.VMSSName),
Properties: &armcompute.VirtualMachineScaleSetNetworkConfigurationProperties{
Primary: to.Ptr(true),
EnableIPForwarding: to.Ptr(true),
IPConfigurations: []*armcompute.VirtualMachineScaleSetIPConfiguration{
{
Name: to.Ptr(fmt.Sprintf("%s0", name)),
Name: to.Ptr(fmt.Sprintf("%s0", s.Runtime.VMSSName)),
Properties: &armcompute.VirtualMachineScaleSetIPConfigurationProperties{
Primary: to.Ptr(true),
LoadBalancerBackendAddressPools: []*armcompute.SubResource{
Expand All @@ -451,13 +478,13 @@ func getBaseVMSSModel(name, sshPublicKey, customData, cseCmd string, cluster *Cl
fmt.Sprintf(
loadBalancerBackendAddressPoolIDTemplate,
config.Config.SubscriptionID,
*cluster.Model.Properties.NodeResourceGroup,
*s.Runtime.Cluster.Model.Properties.NodeResourceGroup,
),
),
},
},
Subnet: &armcompute.APIEntityReference{
ID: to.Ptr(cluster.SubnetID),
ID: to.Ptr(s.Runtime.Cluster.SubnetID),
},
},
},
Expand Down Expand Up @@ -488,6 +515,19 @@ func getBaseVMSSModel(name, sshPublicKey, customData, cseCmd string, cluster *Cl
},
}
}
if s.VHD.Windows() {
model.Identity = &armcompute.VirtualMachineScaleSetIdentity{
Type: to.Ptr(armcompute.ResourceIdentityTypeSystemAssignedUserAssigned),
UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{
config.Config.VMIdentityResourceID(): {},
},
}
model.Properties.VirtualMachineProfile.StorageProfile.OSDisk.OSType = to.Ptr(armcompute.OperatingSystemTypesWindows)
model.Properties.VirtualMachineProfile.OSProfile.LinuxConfiguration = nil
model.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.Publisher = to.Ptr("Microsoft.Compute")
model.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.Type = to.Ptr("CustomScriptExtension")
model.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.TypeHandlerVersion = to.Ptr("1.10")
}
return model
}

Expand Down

0 comments on commit 6fab59f

Please sign in to comment.