Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Nextflow integration test to deployer #782

Merged
merged 7 commits into from
Sep 24, 2024
Merged

Add Nextflow integration test to deployer #782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
111f089
add integration tests to deployer
MattMcL4475 Sep 6, 2024
16bed5c
add nextflow int tests
MattMcL4475 Sep 16, 2024
ef7bf70
Merge branch 'main' into feature/addIntegrationTestsToDeployer
BMurri Sep 23, 2024
62da386
Get nextflow run hello working
BMurri Sep 24, 2024
ae27e31
Clarify meaning via variable names
BMurri Sep 24, 2024
b7b78c5
Replace hello with nf-canary
BMurri Sep 24, 2024
ffcd758
Remove nextflow tes.config after nextflow integrarion test and refactor
BMurri Sep 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/deploy-tes-on-azure/Configuration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public abstract class UserAccessibleConfiguration
public int AksPoolSize { get; set; } = 2;
public bool? CrossSubscriptionAKSDeployment { get; set; } = null;
public bool Silent { get; set; }
public bool RunIntTests { get; set; }
public bool DeleteResourceGroupOnFailure { get; set; }
public string TesImageName { get; set; }
public bool SkipTestWorkflow { get; set; } = false;
Expand Down
167 changes: 147 additions & 20 deletions src/deploy-tes-on-azure/Deployer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
using System.Net.Http;
using System.Net.WebSockets;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Azure;
Expand Down Expand Up @@ -258,14 +259,14 @@ await Execute("Connecting to Azure Services...", async () =>
kubernetesManager.TesHostname = tesHostname;
configuration.EnableIngress = bool.TryParse(enableIngress, out var parsed) ? parsed : null;

var tesCredentials = new FileInfo(Path.Combine(Directory.GetCurrentDirectory(), TesCredentialsFileName));
tesCredentials.Refresh();
var tesCredentialsFile = new FileInfo(Path.Combine(Directory.GetCurrentDirectory(), TesCredentialsFileName));
tesCredentialsFile.Refresh();

if (configuration.EnableIngress.GetValueOrDefault() && tesCredentials.Exists)
if (configuration.EnableIngress.GetValueOrDefault() && tesCredentialsFile.Exists)
{
try
{
using var stream = tesCredentials.OpenRead();
using var stream = tesCredentialsFile.OpenRead();
var (hostname, tesUsername, tesPassword) = TesCredentials.Deserialize(stream);

if (kubernetesManager.TesHostname.Equals(hostname, StringComparison.InvariantCultureIgnoreCase) && string.IsNullOrEmpty(configuration.TesPassword))
Expand Down Expand Up @@ -649,11 +650,13 @@ await Execute(
});
}

TesCredentials tesCredentials = default;

if (configuration.OutputTesCredentialsJson.GetValueOrDefault())
{
// Write credentials to JSON file in working directory
var credentialsJson = new TesCredentials(kubernetesManager.TesHostname, configuration.TesUsername, configuration.TesPassword)
.Serialize();
tesCredentials = new TesCredentials(kubernetesManager.TesHostname, configuration.TesUsername, configuration.TesPassword);
var credentialsJson = tesCredentials.Serialize();

var credentialsPath = Path.Combine(Directory.GetCurrentDirectory(), TesCredentialsFileName);
await File.WriteAllTextAsync(credentialsPath, credentialsJson, cts.Token);
Expand Down Expand Up @@ -706,26 +709,47 @@ await Execute(

var portForwardTask = startPortForward(tokenSource.Token);
await Task.Delay(longRetryWaitTime * 2, tokenSource.Token); // Give enough time for kubectl to standup the port forwarding.
var runTestTask = RunTestTaskAsync("localhost:8088", isPreemptible: batchAccountData.LowPriorityCoreQuota > 0);
var testsToRun = Enumerable.Empty<Func<Task<bool>>>()
.Append(() => RunTestTaskAsync(
"localhost:8088",
isPreemptible: batchAccountData.LowPriorityCoreQuota > 0));

for (var task = await Task.WhenAny(portForwardTask, runTestTask);
runTestTask != task;
task = await Task.WhenAny(portForwardTask, runTestTask))
if (configuration.RunIntTests)
{
try
{
await portForwardTask;
}
catch (Exception ex)
testsToRun = testsToRun.Append(() => RunIntegrationTestsAsync(
"localhost:8088",
tesCredentials,
isPreemptible: !(batchAccountData.DedicatedCoreQuota >= 2 && maxPerFamilyQuota.Append(0).Max() >= 2),
storageAccount,
managedIdentity.Data.ClientId?.ToString("D")));
}

var isTestWorkflowSuccessful = true;

foreach (var testFactory in testsToRun)
{
var runTestTask = testFactory();

for (var task = await Task.WhenAny(portForwardTask, runTestTask);
isTestWorkflowSuccessful && runTestTask != task;
task = await Task.WhenAny(portForwardTask, runTestTask))
{
ConsoleEx.WriteLine($"kubectl stopped unexpectedly ({ex.Message}).", ConsoleColor.Red);
try
{
await portForwardTask;
}
catch (Exception ex)
{
ConsoleEx.WriteLine($"kubectl stopped unexpectedly ({ex.Message}).", ConsoleColor.Red);
}

ConsoleEx.WriteLine($"Restarting kubectl...");
portForwardTask = startPortForward(tokenSource.Token);
}

ConsoleEx.WriteLine($"Restarting kubectl...");
portForwardTask = startPortForward(tokenSource.Token);
isTestWorkflowSuccessful &= await runTestTask;
}

var isTestWorkflowSuccessful = await runTestTask;
exitCode = isTestWorkflowSuccessful ? 0 : 1;

if (!isTestWorkflowSuccessful)
Expand Down Expand Up @@ -844,12 +868,75 @@ private async Task PerformHelmDeploymentAsync(ContainerServiceManagedClusterReso
}
}

private async ValueTask<BlobClient> CreateNextflowConfig(TesCredentials tesCredentials, StorageAccountResource storageAccount, string userAssignedManagedIdentityClientId)
{
StringBuilder sb = new();
sb.AppendLine(@"plugins {");
sb.AppendLine(@" id 'nf-ga4gh'");
sb.AppendLine(@"}");
sb.AppendLine(@"process {");
sb.AppendLine(@" executor = 'tes'");
sb.AppendLine(@"}");
sb.AppendLine(@"azure {");
sb.AppendLine(@" managedIdentity {");
sb.AppendLine($" clientId='{userAssignedManagedIdentityClientId}'");
sb.AppendLine(@" }");
sb.AppendLine(@" storage {");
sb.AppendLine($" accountName='{storageAccount.Id.Name}'");
sb.AppendLine(@" }");
sb.AppendLine(@"}");
sb.AppendLine($"tes.endpoint='https://{tesCredentials.TesHostname}'");
sb.AppendLine($"tes.basicUsername='{tesCredentials.TesUsername}'");
sb.AppendLine($"tes.basicPassword='{tesCredentials.TesPassword}'");
sb.AppendLine(@"process {");
sb.AppendLine(@" container='docker.io/library/ubuntu:latest'");
sb.AppendLine(@"}");
var configText = sb.ToString();
var tesConfig = GetBlobClient(storageAccount.Data, InputsContainerName, "test/nextflow/tes.config");
await UploadTextToStorageAccountAsync(tesConfig, configText, cts.Token);
return tesConfig;
}

[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1859:Use concrete types when possible for improved performance", Justification = "We are explicitly using the contract specified in the ITesClient interface.")]
private async Task<bool> RunNextflowTaskAsync(string tesHostname, bool isPreemptible, StorageAccountResource storageAccount, BlobClient tesConfig)
{
TesTask testTesTask = new();
testTesTask.Resources.Preemptible = isPreemptible;
testTesTask.Resources.CpuCores = 2;
testTesTask.Resources.RamGb = 32;
testTesTask.Resources.DiskGb = 100;

testTesTask.Executors.Add(new()
{
//Image = "nextflow/nextflow:24.04.4",
Image = "nextflow/nextflow:24.08.0-edge",
Command = ["/bin/sh", "-c", "nextflow run seqeralabs/nf-canary -r main -c /tmp/tes.config -w 'az://outputs' || cat .nextflow.log"],
});

testTesTask.Inputs.Add(new()
{
Path = "/tmp/tes.config",
Url = tesConfig.Uri.ToString()
});

using ITesClient tesClient = new TesClient(new($"http://{tesHostname}"));
var completedTask = await tesClient.CreateAndWaitTilDoneAsync(testTesTask, cts.Token);
ConsoleEx.WriteLine($"TES Task State: {completedTask.State}");

if (completedTask.State != TesState.COMPLETE)
{
ConsoleEx.WriteLine($"Failure reason: {completedTask.FailureReason}");
}

return completedTask.State == TesState.COMPLETE;
}

[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1859:Use concrete types when possible for improved performance", Justification = "We are explicitly using the contract specified in the ITesClient interface.")]
private async Task<bool> RunTesTaskImplAsync(string tesHostname, bool isPreemptible)
{
TesTask testTesTask = new();
testTesTask.Resources.Preemptible = isPreemptible;
testTesTask.Executors.Add(new TesExecutor
testTesTask.Executors.Add(new()
{
Image = "ubuntu",
Command = ["/bin/sh", "-c", "cat /proc/sys/kernel/random/uuid"],
Expand Down Expand Up @@ -894,6 +981,46 @@ private async Task<bool> RunTestTaskAsync(string tesEndpoint, bool isPreemptible
return isTestWorkflowSuccessful;
}

private async Task<bool> RunNextflowIntegrationTestAsync(string tesEndpoint, TesCredentials tesCredentials, bool isPreemptible, StorageAccountResource storageAccount, string userAssignedManagedIdentityClientId)
{
var tesConfig = await CreateNextflowConfig(tesCredentials, storageAccount, userAssignedManagedIdentityClientId);

try
{
return await RunNextflowTaskAsync(tesEndpoint, isPreemptible, storageAccount, tesConfig);
}
finally
{
await tesConfig.DeleteIfExistsAsync(cancellationToken: CancellationToken.None);
}
}

private async Task<bool> RunIntegrationTestsAsync(string tesEndpoint, TesCredentials tesCredentials, bool isPreemptible, StorageAccountResource storageAccount, string userAssignedManagedIdentityClientId)
{
var startTime = DateTime.UtcNow;
var line = ConsoleEx.WriteLine("Running integration tests...");
// TODO: Add more integration tests here
var isTestWorkflowSuccessful = await RunNextflowIntegrationTestAsync(tesEndpoint, tesCredentials, isPreemptible, storageAccount, userAssignedManagedIdentityClientId);
WriteExecutionTime(line, startTime);

if (isTestWorkflowSuccessful)
{
ConsoleEx.WriteLine();
ConsoleEx.WriteLine($"Integration test tasks succeeded.", ConsoleColor.Green);
ConsoleEx.WriteLine();
}
else
{
ConsoleEx.WriteLine();
ConsoleEx.WriteLine($"Integration test tasks failed.", ConsoleColor.Red);
ConsoleEx.WriteLine();
WriteGeneralRetryMessageToConsole();
ConsoleEx.WriteLine();
}

return isTestWorkflowSuccessful;
}

private async Task<KeyVaultResource> ValidateAndGetExistingKeyVaultAsync()
{
if (string.IsNullOrWhiteSpace(configuration.KeyVaultName))
Expand Down
Loading