Update sample projects to support variable slots scheduling (#293)

* Update sample projects to support variable slots scheduling

* Refactor task slots per task and add it into output

* Bump Microsoft.Azure.Batch package to 14.0.0

CSharp/ArticleProjects/ApplicationInsights/TopNWords/TopNWords.csproj

@@ -10,11 +10,11 @@
     <ProjectReference Include="..\..\..\Common\Microsoft.Azure.Batch.Samples.Common.csproj" />
     <ProjectReference Include="..\Microsoft.Azure.Batch.Samples.TelemetryInitializer\Microsoft.Azure.Batch.Samples.TelemetryInitializer.csproj" />
     <ProjectReference Include="..\Microsoft.Azure.Batch.Samples.TelemetryStartTask\Microsoft.Azure.Batch.Samples.TelemetryStartTask.csproj" />
-    
+
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
     <PackageReference Include="Microsoft.Azure.Batch.FileStaging" Version="8.3.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="2.1.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.0" />

CSharp/ArticleProjects/EfficientListQueries/EfficientListQueries.csproj

@@ -11,7 +11,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
   </ItemGroup>
 
 </Project>

CSharp/ArticleProjects/EfficientListQueries/Program.cs

@@ -47,7 +47,7 @@ private static async Task MainAsync(string[] args)
             // You may adjust these values to experiment with different compute resource scenarios.
             const string nodeSize     = "standard_d1_v2";
             const int nodeCount       = 1;
-            const int maxTasksPerNode = 4;
+            const int taskSlotsPerNode = 4;
 
             // Adjust the task count to experiment with different list operation query durations
             const int taskCount = 5000;
@@ -72,8 +72,8 @@ private static async Task MainAsync(string[] args)
                     poolId,
                     nodeSize,
                     nodeCount,
-                    maxTasksPerNode);
-                
+                    taskSlotsPerNode);
+
                 // Create a CloudJob, or obtain an existing job with the specified ID
                 CloudJob job = await ArticleHelpers.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
 
@@ -190,9 +190,9 @@ private static async Task QueryTasksAsync(BatchClient batchClient, string jobId,
             Stopwatch stopwatch = Stopwatch.StartNew();
 
             taskList.AddRange(await batchClient.JobOperations.ListTasks(jobId, detail).ToListAsync());
-            
+
             stopwatch.Stop();
-            
+
             Console.WriteLine("{0} tasks retrieved in {1} (ExpandClause: {2} | FilterClause: {3} | SelectClause: {4})",
                 taskList.Count,
                 stopwatch.Elapsed,

CSharp/ArticleProjects/JobPrepRelease/JobPrepRelease.csproj

@@ -11,7 +11,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
   </ItemGroup>
 
 </Project>

CSharp/ArticleProjects/MultiInstanceTasks/MultiInstanceTasks.csproj

@@ -23,7 +23,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
     <PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
   </ItemGroup>
 

CSharp/ArticleProjects/MultiInstanceTasks/Program.cs

@@ -39,7 +39,7 @@ public static void Main(string[] args)
                 Console.ReadLine();
             }
         }
-        
+
         public static async Task MainAsync()
         {
             const string poolId = "MultiInstanceSamplePool";
@@ -206,7 +206,7 @@ private static async Task CreatePoolAsync(BatchClient batchClient, string poolId
             // sample, MPIHelloWorld.exe) running on the different nodes
             unboundPool.InterComputeNodeCommunicationEnabled = true;
             // REQUIRED for multi-instance tasks
-            unboundPool.MaxTasksPerComputeNode = 1;
+            unboundPool.TaskSlotsPerNode = 1;
 
             // Specify the application and version to deploy to the compute nodes.
             unboundPool.ApplicationPackageReferences = new List<ApplicationPackageReference>

CSharp/ArticleProjects/ParallelTasks/ParallelTasks.csproj

@@ -11,7 +11,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
   </ItemGroup>
 
 </Project>

CSharp/ArticleProjects/ParallelTasks/Program.cs

@@ -45,13 +45,13 @@ public static void Main(string[] args)
         private static async Task MainAsync(string[] args)
         {
             // You may adjust these values to experiment with different compute resource scenarios.
-            const string nodeSize     = "standard_d1_v2";
-            const int nodeCount       = 4;
-            const int maxTasksPerNode = 4;
-            const int taskCount       = 32;
+            const string nodeSize      = "standard_d1_v2";
+            const int nodeCount        = 4;
+            const int taskSlotsPerNode = 4;
+            const int taskCount        = 32;
 
             // Ensure there are enough tasks to help avoid hitting some timeout conditions below
-            int minimumTaskCount = nodeCount * maxTasksPerNode * 2;
+            int minimumTaskCount = nodeCount * taskSlotsPerNode * 2;
             if (taskCount < minimumTaskCount)
             {
                 Console.WriteLine("You must specify at least two tasks per node core for this sample ({0} tasks in this configuration).", minimumTaskCount);
@@ -61,6 +61,18 @@ private static async Task MainAsync(string[] args)
                 return;
             }
 
+            // In this sample, each task is assigned with random required slots; adjust these values
+            // to simulate variable task slots together with allowed task slots per compute node
+            const int maxTaskSlots     = 2;
+            if (maxTaskSlots > taskSlotsPerNode)
+            {
+                Console.WriteLine("Invalid task slot configuration: maxTaskSlots for task should not be greater than pool's TaskSlotsPerNode");
+                Console.WriteLine();
+
+                // Invalid task slot configuration, exit the application
+                return;
+            }
+
             // In this sample, the tasks simply ping localhost on the compute nodes; adjust these
             // values to simulate variable task duration
             const int minPings = 30;
@@ -89,7 +101,7 @@ private static async Task MainAsync(string[] args)
                     poolId,
                     nodeSize,
                     nodeCount,
-                    maxTasksPerNode);
+                    taskSlotsPerNode);
 
                 // Create a CloudJob, or obtain an existing pool with the specified ID
                 CloudJob job = await ArticleHelpers.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
@@ -102,7 +114,10 @@ private static async Task MainAsync(string[] args)
                 {
                     string taskId = "task" + i.ToString().PadLeft(3, '0');
                     string taskCommandLine = "ping -n " + rand.Next(minPings, maxPings + 1).ToString() + " localhost";
-                    CloudTask task = new CloudTask(taskId, taskCommandLine);
+                    CloudTask task = new CloudTask(taskId, taskCommandLine)
+                    {
+                        RequiredSlots = rand.Next(1, maxTaskSlots + 1)
+                    };
                     tasks.Add(task);
                 }
 
@@ -129,6 +144,18 @@ private static async Task MainAsync(string[] args)
                 await GettingStartedCommon.PrintNodeTasksAsync(batchClient, pool.Id);
                 Console.WriteLine();
 
+                // Print out running task and task slot counts on all nodes.
+                Console.WriteLine();
+                await GettingStartedCommon.PrintNodeTaskCountsAsync(batchClient, pool.Id);
+                Console.WriteLine();
+
+                // Print out task and task slot counts of the job.
+                // Note: it may have delay as Batch service is aggregating task counts for the job.
+                await Task.Delay(TimeSpan.FromSeconds(5));
+                Console.WriteLine();
+                await GettingStartedCommon.PrintJobTaskCountsAsync(batchClient, jobId);
+                Console.WriteLine();
+
                 // Pause execution while we wait for all of the tasks to complete
                 Console.WriteLine("Waiting for task completion...");
                 Console.WriteLine();
@@ -147,10 +174,16 @@ await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
 
                 stopwatch.Stop();
 
+                // Print out job task counts again.
+                await Task.Delay(TimeSpan.FromSeconds(5));
+                Console.WriteLine();
+                await GettingStartedCommon.PrintJobTaskCountsAsync(batchClient, jobId);
+                Console.WriteLine();
+
                 // Obtain the tasks, specifying a detail level to limit the number of properties returned for each task.
                 // If you have a large number of tasks, specifying a DetailLevel is extremely important in reducing the
                 // amount of data transferred, lowering your query response times in increasing performance.
-                ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id,commandLine,nodeInfo,state");
+                ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id,commandLine,nodeInfo,state,requiredSlots");
                 IPagedEnumerable<CloudTask> allTasks = batchClient.JobOperations.ListTasks(job.Id, detail);
 
                 // Get a collection of the completed tasks sorted by the compute nodes on which they executed
@@ -173,7 +206,7 @@ await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
 
                     lastNodeId = task.ComputeNodeInformation.ComputeNodeId;
 
-                    Console.WriteLine("\t{0}: {1}", task.Id, task.CommandLine);
+                    Console.WriteLine($"\t{task.Id} (slots={task.RequiredSlots}): {task.CommandLine}");
                 }
 
                 // Get a collection of the uncompleted tasks which may exist if the TaskMonitor timeout was hit
@@ -200,11 +233,12 @@ await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
 
                 // Print some summary information
                 Console.WriteLine();
-                Console.WriteLine("             Nodes: " + nodeCount);
-                Console.WriteLine("         Node size: " + nodeSize);
-                Console.WriteLine("Max tasks per node: " + pool.MaxTasksPerComputeNode);
-                Console.WriteLine("             Tasks: " + tasks.Count);
-                Console.WriteLine("          Duration: " + stopwatch.Elapsed);
+                Console.WriteLine("              Nodes: " + nodeCount);
+                Console.WriteLine("          Node size: " + nodeSize);
+                Console.WriteLine("Task slots per node: " + pool.TaskSlotsPerNode);
+                Console.WriteLine(" Max slots per task: " + maxTaskSlots);
+                Console.WriteLine("              Tasks: " + tasks.Count);
+                Console.WriteLine("           Duration: " + stopwatch.Elapsed);
                 Console.WriteLine();
                 Console.WriteLine("Done!");
                 Console.WriteLine();

CSharp/ArticleProjects/ParallelTasks/README.md

@@ -1,3 +1,7 @@
 ### ParallelTasks
 
-This console application sample project backs the code snippets found in [Maximize Azure Batch compute resource usage with concurrent node tasks](http://azure.microsoft.com/documentation/articles/batch-parallel-node-tasks/). The application demonstrates the creation of a Batch pool whose compute nodes are configured for executing multiple concurrent tasks, and prints node and task information to the console during execution to show how tasks are distributed among compute nodes and node cores.
+This console application sample project backs the code snippets found in [Maximize Azure Batch compute resource usage with concurrent node tasks](http://azure.microsoft.com/documentation/articles/batch-parallel-node-tasks/). The application demonstrates the creation of a Batch pool whose compute nodes are configured for executing multiple concurrent tasks, submit tasks with variable slots, and prints node/job/task information to the console during execution to show how tasks are distributed among compute nodes and node cores.
+
+1. Open the project in **Visual Studio 2017**.
+2. Add your Batch and Storage **account credentials** to **accountsettings.json** in the Microsoft.Azure.Batch.Samples.Common project.
+3. **Build** and then **run** the solution. Restore any NuGet packages if prompted.

CSharp/ArticleProjects/PersistOutputs/PersistOutputs.csproj

@@ -17,9 +17,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
     <PackageReference Include="Microsoft.Azure.Batch.Conventions.Files" Version="3.5.1" />
-  
+
   </ItemGroup>
 
 </Project>

CSharp/ArticleProjects/PersistOutputs/PersistOutputsTask/PersistOutputsTask.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
     <PackageReference Include="Microsoft.Azure.Batch.Conventions.Files" Version="3.5.1" />
   </ItemGroup>
 

CSharp/ArticleProjects/TaskDependencies/TaskDependencies.csproj

@@ -11,7 +11,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
   </ItemGroup>
 
 </Project>

CSharp/BatchMetrics/BatchMetrics/BatchMetrics.csproj

@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
   </ItemGroup>
 
 </Project>

CSharp/BatchMetrics/BatchMetricsUsageSample/BatchMetricsUsageSample.csproj

@@ -12,7 +12,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
   </ItemGroup>
 
 </Project>

CSharp/BatchMetrics/BatchMetricsUsageSample/JobSubmitter.cs

@@ -31,7 +31,7 @@ public class JobSubmitter
         private static readonly TimeSpan JobTaskTimeoutIncrement = TimeSpan.FromSeconds(10);
         private const string JobTaskIdPrefix = "testtask-";
         private static readonly TimeSpan JobInterval = TimeSpan.FromMinutes(2);
-      
+
         public JobSubmitter(BatchClient batchClient)
         {
             this.batchClient = batchClient;
@@ -48,7 +48,7 @@ private async Task CreatePoolAsync()
                 virtualMachineSize: PoolNodeSize,
                 cloudServiceConfiguration: new CloudServiceConfiguration(PoolOSFamily));
 
-            pool.MaxTasksPerComputeNode = 2;
+            pool.TaskSlotsPerNode = 2;
 
             var createPoolResult = await GettingStartedCommon.CreatePoolIfNotExistAsync(this.batchClient, pool);
 

CSharp/Common/ArticleHelpers.cs

@@ -21,22 +21,22 @@ public static class ArticleHelpers
         /// <param name="poolId">The ID of the <see cref="CloudPool"/>.</param>
         /// <param name="nodeSize">The size of the nodes within the pool.</param>
         /// <param name="nodeCount">The number of nodes to create within the pool.</param>
-        /// <param name="maxTasksPerNode">The maximum number of tasks to run concurrently on each node.</param>
+        /// <param name="taskSlotsPerNode">The number of task slots to run concurrent tasks on each node.</param>
         /// <returns>A bound <see cref="CloudPool"/> with the specified properties.</returns>
-        public async static Task<CloudPool> CreatePoolIfNotExistAsync(BatchClient batchClient, string poolId, string nodeSize, int nodeCount, int maxTasksPerNode)
+        public async static Task<CloudPool> CreatePoolIfNotExistAsync(BatchClient batchClient, string poolId, string nodeSize, int nodeCount, int taskSlotsPerNode)
         {
             // Create and configure an unbound pool with the specified ID
             CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
                 virtualMachineSize: nodeSize,
                 targetDedicatedComputeNodes: nodeCount,
                 cloudServiceConfiguration: new CloudServiceConfiguration("5"));
-
-            pool.MaxTasksPerComputeNode = maxTasksPerNode;
 
-            // We want each node to be completely filled with tasks (i.e. up to maxTasksPerNode) before
+            pool.TaskSlotsPerNode = taskSlotsPerNode;
+
+            // We want each node to be completely filled with tasks (i.e. up to taskSlotsPerNode) before
             // tasks are assigned to the next node in the pool
             pool.TaskSchedulingPolicy = new TaskSchedulingPolicy(ComputeNodeFillType.Pack);
-            
+
             await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool).ConfigureAwait(continueOnCapturedContext: false);
 
             return await batchClient.PoolOperations.GetPoolAsync(poolId).ConfigureAwait(continueOnCapturedContext: false);

CSharp/Common/GettingStartedCommon.cs

@@ -94,6 +94,53 @@ await nodes.ForEachAsync(node =>
             Console.WriteLine("==================");
         }
 
+        /// <summary>
+        /// Prints running task and task slot counts to the console for each of the nodes in the specified pool.
+        /// </summary>
+        /// <param name="batchClient">The Batch client.</param>
+        /// <param name="poolId">The ID of the <see cref="CloudPool"/> containing the nodes whose task information should be printed to the console.</param>
+        /// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
+        public static async Task PrintNodeTaskCountsAsync(BatchClient batchClient, string poolId)
+        {
+            Console.WriteLine("Listing Node Running Task Counts");
+            Console.WriteLine("==================");
+
+            ODATADetailLevel nodeDetail = new ODATADetailLevel(selectClause: "id,runningTasksCount,runningTaskSlotsCount");
+            IPagedEnumerable<ComputeNode> nodes = batchClient.PoolOperations.ListComputeNodes(poolId, nodeDetail);
+
+            await nodes.ForEachAsync(node =>
+            {
+                Console.WriteLine();
+                Console.WriteLine(node.Id + " :");
+                Console.WriteLine($"RunningTasks = {node.RunningTasksCount}, RunningTaskSlots = {node.RunningTaskSlotsCount}");
+                
+            }).ConfigureAwait(continueOnCapturedContext: false);
+
+            Console.WriteLine("==================");
+        }
+
+        /// <summary>
+        /// Prints task and task slot counts per task state for the specified job.
+        /// </summary>
+        /// <param name="batchClient">The Batch client.</param>
+        /// <param name="poolId">The ID of the <see cref="CloudJob"/>.</param>
+        /// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
+        public static async Task PrintJobTaskCountsAsync(BatchClient batchClient, string jobId)
+        {
+            Console.WriteLine("Listing Job Task Counts");
+            Console.WriteLine("==================");
+
+            TaskCountsResult result = await batchClient.JobOperations.GetJobTaskCountsAsync(jobId);
+
+            Console.WriteLine();
+            Console.WriteLine(jobId + " :");
+            Console.WriteLine("\t\tActive\tRunning\tCompleted");
+            Console.WriteLine($"TaskCounts:\t{result.TaskCounts.Active}\t{result.TaskCounts.Running}\t{result.TaskCounts.Completed}");
+            Console.WriteLine($"TaskSlotCounts:\t{result.TaskSlotCounts.Active}\t{result.TaskSlotCounts.Running}\t{result.TaskSlotCounts.Completed}");
+
+            Console.WriteLine("==================");
+        }
+
         public static string CreateJobId(string prefix)
         {
             // a job is uniquely identified by its ID so your account name along with a timestamp is added as suffix

CSharp/Common/Microsoft.Azure.Batch.Samples.Common.csproj

@@ -3,7 +3,7 @@
     <TargetFramework>net462</TargetFramework>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
     <PackageReference Include="Microsoft.Azure.Batch.FileStaging" Version="8.3.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="2.1.0" />