fix(deadline): Windows Workers fail to deploy waiting for Deadline launcher service to restart (#354)

Fixes #353 and #312

Author: jusiskin

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
       uses: actions/[email protected]
       with:
         node-version: ${{ matrix.node-version }}
+    - run: npm install --global yarn
     - run: yarn global add typescript
-    - run: yarn install --frozen-lockfile
-    - run: ./build.sh
+    - run: yarn build
     - run: ./pack.sh

packages/aws-rfdk/lib/deadline/lib/render-queue-ref.ts

@@ -361,4 +361,11 @@ export interface InstanceConnectOptions {
    * The Instance/UserData which will directly connect to the Repository
    */
   readonly host: IHost;
+
+  /**
+   * Whether or not to start or restart the Deadline Launcher after configuring the connection.
+   *
+   * @default true
+   */
+  readonly restartLauncher?: boolean;
 }

packages/aws-rfdk/lib/deadline/lib/rq-connection.ts

@@ -39,10 +39,20 @@ export interface HTTPSConnectionOptions extends BaseConnectionOptions {
 }
 
 interface ConnectionScriptArguments {
-
+  /**
+   * The address of the Render Queue
+   */
   readonly address: string;
 
+  /**
+   * The ARN to the AWS Secrets Manager secret containing the X509 CA Certificate in PEM format.
+   */
   readonly tlsCaArn?: string;
+
+  /**
+   * Whether to restart the Deadline launcher after configuring the Render Queue connection.
+   */
+  readonly restartLauncher?: boolean;
 }
 
 /**
@@ -114,23 +124,31 @@ export abstract class RenderQueueConnection {
         `"\${DEADLINE_PATH}/deadlinecommand" -executeScriptNoGui "${configureScriptPath}" --render-queue "${args.address}" ${dlExtraCommands.join(' ')}`,
         // Cleanup
         `rm -f "${configureScriptPath}"`,
-        'if service --status-all | grep -q "Deadline 10 Launcher"; then',
-        '  service deadline10launcher restart',
-        'fi',
       );
+      if (args.restartLauncher ?? true) {
+        host.userData.addCommands(
+          'if service --status-all | grep -q "Deadline 10 Launcher"; then',
+          '  service deadline10launcher restart',
+          'fi',
+        );
+      }
     } else if ( host.osType === OperatingSystemType.WINDOWS ) {
       host.userData.addCommands(
         '$ErrorActionPreference = "Stop"',
         '$DEADLINE_PATH = (get-item env:"DEADLINE_PATH").Value',
         `& "$DEADLINE_PATH/deadlinecommand.exe" -executeScriptNoGui "${configureScriptPath}" --render-queue "${args.address}" ${dlExtraCommands.join(' ')} 2>&1`,
         `Remove-Item -Path "${configureScriptPath}"`,
-        'If (Get-Service "deadline10launcherservice" -ErrorAction SilentlyContinue) {',
-        '  Restart-Service "deadline10launcherservice"',
-        '} Else {',
-        '  & "$DEADLINE_PATH/deadlinelauncher.exe" -shutdownall 2>&1',
-        '  & "$DEADLINE_PATH/deadlinelauncher.exe" 2>&1',
-        '}',
       );
+      if (args.restartLauncher ?? true) {
+        host.userData.addCommands(
+          'If (Get-Service "deadline10launcherservice" -ErrorAction SilentlyContinue) {',
+          '  Restart-Service "deadline10launcherservice"',
+          '} Else {',
+          '  & "$DEADLINE_PATH/deadlinelauncher.exe" -shutdownall 2>&1',
+          '  & "$DEADLINE_PATH/deadlinelauncher.exe" -nogui 2>&1',
+          '}',
+        );
+      }
     }
   }
 
@@ -163,6 +181,7 @@ class HTTPConnection extends RenderQueueConnection {
       params.host,
       {
         address: `http://${this.config.endpoint.socketAddress}`,
+        restartLauncher: params.restartLauncher,
       },
     );
   }
@@ -200,6 +219,7 @@ class HTTPSConnection extends RenderQueueConnection {
       {
         address: `https://${this.config.endpoint.socketAddress}`,
         tlsCaArn: this.config.caCert.secretArn,
+        restartLauncher: params.restartLauncher,
       },
     );
 

packages/aws-rfdk/lib/deadline/lib/worker-configuration.ts

@@ -221,7 +221,14 @@ export class WorkerInstanceConfiguration extends Construct {
       );
     }
     props.userDataProvider?.preRenderQueueConfiguration(props.worker);
-    props.renderQueue?.configureClientInstance({ host: props.worker });
+    props.renderQueue?.configureClientInstance({
+      host: props.worker,
+      // Don't restart the Deadline Launcher service after configuring the connection to the Render Queue. We need to
+      // restart it later anyways, and the Windows service for the Deadline Launcher can get locked in the "stopping"
+      // state if you attempt to stop or restart it while it is still restarting. This can cause the user data execution
+      // to get locked waiting for the service to finish stopping/restarting.
+      restartLauncher: false,
+    });
     props.userDataProvider?.preWorkerConfiguration(props.worker);
 
     this.listenerPort = props.workerSettings?.listenerPort ?? WorkerInstanceConfiguration.DEFAULT_LISTENER_PORT;

packages/aws-rfdk/lib/deadline/scripts/bash/configureWorker.sh

@@ -133,7 +133,7 @@ else
   DEADLINE_LAUNCHER="$DEADLINE_PATH/deadlinelauncher"
   "$DEADLINE_LAUNCHER" -shutdownall
   sudo killall -w deadlineworker || true
-  "$DEADLINE_LAUNCHER"
+  "$DEADLINE_LAUNCHER" -nogui
 fi
 
 echo "Script completed successfully."

packages/aws-rfdk/lib/deadline/scripts/powershell/configureWorker.ps1

@@ -127,7 +127,7 @@ If (Get-Service $serviceName -ErrorAction SilentlyContinue) {
     $DEADLINE_LAUNCHER = $DEADLINE_PATH + '/deadlinelauncher.exe'
     & $DEADLINE_LAUNCHER -shutdownall | Out-Default
     taskkill /f /fi "IMAGENAME eq deadlineworker.exe"
-    & $DEADLINE_LAUNCHER
+    & $DEADLINE_LAUNCHER -nogui
 }
 
 Write-Host "Script completed successfully."

packages/aws-rfdk/lib/deadline/test/asset-constants.ts

@@ -18,8 +18,8 @@ export {
 
 // configureWorker.sh
 export const CONFIG_WORKER_ASSET_LINUX = {
-  Bucket: 'AssetParameters21c2af3bc1d4fd78061765b059dcc8e32568828e5cf479b08115489651491c8fS3BucketF10C60A7',
-  Key: 'AssetParameters21c2af3bc1d4fd78061765b059dcc8e32568828e5cf479b08115489651491c8fS3VersionKey7FDCC89A',
+  Bucket: 'AssetParameters1cfdffe73bb016717ba1f43d64fe528af27b3784f524a97bb36533a6e6d057ffS3Bucket9FBDD688',
+  Key: 'AssetParameters1cfdffe73bb016717ba1f43d64fe528af27b3784f524a97bb36533a6e6d057ffS3VersionKey02A3157B',
 };
 
 // configureWorker.ps1
@@ -29,8 +29,8 @@ export const CONFIG_WORKER_ASSET_WINDOWS = {
 };
 
 export const CONFIG_WORKER_PORT_ASSET_WINDOWS = {
-  Bucket: 'AssetParameters0b4fe3ffb7177773bb2781f92b37d9b01b3bd37ee60ea1715c0ad407f141005dS3BucketE7B32C3E',
-  Key: 'AssetParameters0b4fe3ffb7177773bb2781f92b37d9b01b3bd37ee60ea1715c0ad407f141005dS3VersionKey843794E3',
+  Bucket: 'AssetParameters3227efc256da3ae31791b7c80e1532cac975116846f179f118a20843e0c2ee80S3Bucket6583BE37',
+  Key: 'AssetParameters3227efc256da3ae31791b7c80e1532cac975116846f179f118a20843e0c2ee80S3VersionKey6C80977B',
 };
 
 // installDeadlineRepository.sh

packages/aws-rfdk/lib/deadline/test/render-queue.test.ts

@@ -1227,7 +1227,7 @@ describe('RenderQueue', () => {
               '  Restart-Service "deadline10launcherservice"\n' +
               '} Else {\n' +
               '  & "$DEADLINE_PATH/deadlinelauncher.exe" -shutdownall 2>&1\n' +
-              '  & "$DEADLINE_PATH/deadlinelauncher.exe" 2>&1\n' +
+              '  & "$DEADLINE_PATH/deadlinelauncher.exe" -nogui 2>&1\n' +
               '}</powershell>',
             ],
           ],
@@ -1635,7 +1635,7 @@ describe('RenderQueue', () => {
               '  Restart-Service "deadline10launcherservice"\n' +
               '} Else {\n' +
               '  & "$DEADLINE_PATH/deadlinelauncher.exe" -shutdownall 2>&1\n' +
-              '  & "$DEADLINE_PATH/deadlinelauncher.exe" 2>&1\n' +
+              '  & "$DEADLINE_PATH/deadlinelauncher.exe" -nogui 2>&1\n' +
               '}</powershell>',
             ],
           ],

packages/aws-rfdk/lib/deadline/test/worker-fleet.test.ts

@@ -721,9 +721,6 @@ test('default worker fleet is created correctly custom subnet values', () => {
           ],
         },
         '\"\n' +
-        'if service --status-all | grep -q "Deadline 10 Launcher"; then\n' +
-        '  service deadline10launcher restart\n' +
-        'fi\n' +
         "mkdir -p $(dirname '/tmp/",
         {
           'Fn::Select': [
@@ -1214,9 +1211,6 @@ test('default worker fleet is created correctly with groups, pools and region',
         ],
       },
       '\"\n' +
-      'if service --status-all | grep -q "Deadline 10 Launcher"; then\n' +
-      '  service deadline10launcher restart\n' +
-      'fi\n' +
       "mkdir -p $(dirname '/tmp/",
       {
         'Fn::Select': [