feat: Option to register runners on organization instead of repo (#451)

**App.svelte** - [x] Add ui component - [x] Add parameter to new app request - [x] Add parameter to existing app request **Secrets.ts** - [x] Create new secret for runner registration level in cdk **Setup.lambda.ts** - [x] Add secret arn to setup lambda.ts - [x] Determine registration level based on permissions for newapp - [x] Determine registration level for existing app - [x] Update secret value **Tokenretriever.lambda.ts** - [x] Add env variable for registration level / secret arn - [x] Add new condition to register token from org (octokit) Determine runner registration config command for: - [x] ec2 linux(userdata) - [x] ec2 windows(userdata) - [x] fargate linux - [x] lambda linux - [x] lambda linux arm - [x] codebuild linux - [x] codebuild linux arm - [x] ecs linux - [x] ecs linux arm - [x] ec2 linux arm - [x] ... Tests: - [x] create new app with org level registration - [x] create with existing app with org level registration - [x] run default integration test to setup solution - [x] run github action runners with all providers in default integration test in github.com result: ![image](https://github.com/CloudSnorkel/cdk-github-runners/assets/5619511/9668c5d2-b303-4cc7-98aa-304c1586d364) closes #442
CloudSnorkel · Nov 1, 2023 · 42c5ade · 42c5ade
1 parent a477f63
commit 42c5ade
Show file tree

Hide file tree

Showing 23 changed files with 401 additions and 177 deletions.
diff --git a/API.md b/API.md
diff --git a/SETUP_GITHUB.md b/SETUP_GITHUB.md
@@ -13,6 +13,7 @@ Integration with GitHub can be done using an [app](#app-authentication) or [pers
 5. If you want to create an app for your organization:
    1. Choose Organization app
    2. Type in the organization slug (ORGANIZATION from https://github.com/ORGANIZATION/REPO)
+   3. Choose registration level for the runners
 6. Click Create GitHub App to take you to GitHub to finish the setup
 7. Follow the instructions on GitHub
 8. When brought back to the setup wizard, click the install link
@@ -27,10 +28,12 @@ Integration with GitHub can be done using an [app](#app-authentication) or [pers
 3. Setup webhook under the webhook section
     1. For Webhook URL use the value of `github.webhook.url` from `status.json`
     2. Open the URL in `github.webhook.secretUrl` from `status.json`, retrieve the secret value, and use it for webhook secret
-4. In the repository permissions section enable:
-    1. Actions: Read and write
-    2. Administration: Read and write
-    3. Deployments: Read-only
+4. In the permissions section enable:
+   1. Repository    -> Actions: Read and write
+   2. Repository    -> Administration: Read and write
+   3. Repository    -> Deployments: Read-only
+   4. Repository    -> Administration: Read and write (only for repository level runners)
+   5. Organization  -> Self-hosted runners: Read and write (only for organization level runners)
 5. In the event subscription section enable:
     1. Workflow job
 6. Under "Where can this GitHub App be installed?" select "Only on this account"
@@ -40,7 +43,8 @@ Integration with GitHub can be done using an [app](#app-authentication) or [pers
 10. Open the URL in `github.auth.secretUrl` from `status.json` and edit the secret value
     1. If you're using a self-hosted GitHub instance, put its domain in `domain` (e.g. `github.mycompany.com`)
     2. Put the new application id in `appId` (e.g. `34789562`)
-    3. Ignore/delete `dummy` and **leave `personalAuthToken` empty**
+    3. If using organization level registration, add `runnerLevel` with `org` as the value
+    4. Ignore/delete `dummy` and **leave `personalAuthToken` empty**
 11. Open the URL in `github.auth.privateKeySecretUrl` from `status.json` and edit the secret value
     1. Open the downloaded private key with any text editor
     2. Copy the text from the private key as-is into the secret

diff --git a/setup/src/App.svelte b/setup/src/App.svelte
@@ -5,31 +5,47 @@
   let domain = 'INSERT_DOMAIN_HERE';
   let auth: undefined | 'newApp' | 'existingApp' | 'pat';
   let appScope: 'user' | 'org' = 'user';
+  let runnerLevel: 'repo' | 'org' = 'repo';
   let org = 'ORGANIZATION';
   let existingAppId: string = '';
   let existingAppPk: string = '';
   let pat: string = '';
   let success: boolean;
   let result: string | undefined;
 
+  interface Permissions {
+    actions: 'write' | 'read';
+    administration?: 'write' | 'read';
+    organization_self_hosted_runners?: 'write' | 'read';
+    deployments: 'write' | 'read';
+  };
+
+  const repositoryPermissions: Permissions = {
+    actions: 'write',
+    administration: 'write',
+    deployments: 'read',
+  };
+
+  const organizationPermissions: Permissions = {
+    actions: 'write',
+    organization_self_hosted_runners: 'write',
+    deployments: 'read',
+  };
+
   const manifest = {
     url: 'https://github.com/CloudSnorkel/cdk-github-runners',
     hook_attributes: {
       url: 'INSERT_WEBHOOK_URL_HERE',
     },
     redirect_url: 'INSERT_BASE_URL_HERE/complete-new-app',
     public: false,
-    default_permissions: {
-      actions: 'write',
-      administration: 'write',
-      deployments: 'read',
-    },
+    default_permissions: <Permissions>repositoryPermissions,
     default_events: [
       'workflow_job',
     ],
   };
 
-  function isSubmitDisabled(instance_, auth_, existingAppId_, existingAppPk_, pat_, success_) {
+  function isSubmitDisabled(instance_, auth_, existingAppId_, existingAppPk_, runnerLevel_, pat_, success_) {
     if (success_) {
       return true;
     }
@@ -40,7 +56,7 @@
       return false;
     }
     if (auth_ === 'existingApp') {
-      return existingAppId_ === '' || existingAppPk_ === '';
+      return existingAppId_ === '' || existingAppPk_ === '' || runnerLevel_ === undefined;
     }
     if (auth_ === 'pat') {
       return pat_ === '';
@@ -89,9 +105,13 @@
 
     function promise(): Promise<string> {
       const rightDomain = instance === 'ghes' ? domain : 'github.com';
+      manifest.default_permissions =
+        runnerLevel === 'repo'
+          ? repositoryPermissions
+          : organizationPermissions;
       switch (auth) {
         case 'newApp':
-          return postJson('domain', { domain: rightDomain })
+          return postJson('domain', { domain: rightDomain, runnerLevel })
             .then(_ => {
               (document.getElementById('appform') as HTMLFormElement).submit();
               return Promise.resolve('Redirecting to GitHub...');
@@ -101,6 +121,7 @@
             appid: existingAppId,
             pk: existingAppPk,
             domain: rightDomain,
+            runnerLevel,
           });
         case 'pat':
           return postJson('pat', {
@@ -232,10 +253,6 @@
         {:else if auth === 'existingApp'}
           <h3>Existing App Details</h3>
           <div class="px-3 py-3">
-            <p>Existing apps must have <code>actions</code> and <code>administration</code> write
-              permissions. Don't forget to set up the webhook and its secret as described in <a
-                href="https://github.com/CloudSnorkel/cdk-github-runners/blob/main/SETUP_GITHUB.md">SETUP_GITHUB.md</a>.
-            </p>
             <div class="form-group row px-3 py-2">
               <label for="appid" class="col-sm-2 col-form-label">App Id</label>
               <div class="col-sm-10">
@@ -248,6 +265,41 @@
                 <textarea class="form-control" id="pk" bind:value={existingAppPk} rows="10"></textarea>
               </div>
             </div>
+            <div class="form-group row px-3 py-2">
+              <div class="col-sm-2 col-form-label">Registration Level</div>
+              <div class="col-sm-10">
+                <div class="form-check">
+                  <input
+                    class="form-check-input"
+                    type="radio"
+                    bind:group={runnerLevel}
+                    value="repo"
+                    id="repo"
+                  />
+                  <label class="form-check-label" for="repo">Repository</label>
+                </div>
+                <div class="form-check">
+                  <input
+                    class="form-check-input"
+                    type="radio"
+                    bind:group={runnerLevel}
+                    value="org"
+                    id="org"
+                  />
+                  <label class="form-check-label" for="org">Organization</label>
+                </div>
+              </div>
+            </div>
+
+            <h4>Required Permissions</h4>
+            <p>The existing app must have the following permissions.</p>
+            <pre>{JSON.stringify(runnerLevel === 'repo' ? repositoryPermissions : organizationPermissions, undefined, 2)}</pre>
+
+            <h4>Webhook</h4>
+            <p>
+              Don't forget to set up the webhook and its secret as described in <a
+                href="https://github.com/CloudSnorkel/cdk-github-runners/blob/main/SETUP_GITHUB.md">SETUP_GITHUB.md</a>.
+            </p>
           </div>
         {:else if auth === 'pat'}
           <h2>Personal Access Token</h2>
@@ -261,6 +313,59 @@
           </div>
         {/if}
 
+        {#if appScope === 'org' && auth === 'newApp'}
+          <h3>Registration Level</h3>
+          <div class="px-3 py-3">
+            <p>
+              Would you like runners to be registered on repository level, or on organization level?
+            </p>
+            <ul>
+              <li>
+                Registering runners on repository level requires the <code>administration</code>
+                <a href="https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/autoscaling-with-self-hosted-runners#authentication-requirements">permission</a>.
+              </li>
+              <li>
+                Registering runners on organization level only requires the <code>organization_self_hosted_runners</code>
+                <a href="https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/autoscaling-with-self-hosted-runners#authentication-requirements">permission</a>
+                which is more fine-grained.
+              </li>
+              <li>
+                Registering runners on organization level means any repository can use them, even if the app wasn't
+                installed on those repositories.
+              </li>
+              <li>
+                Do not use organization level registration if you don't fully trust all repositories in your organization.
+              </li>
+              <li>
+                Use organization level to reduce the permission scope this new app is given.
+              </li>
+              <li>
+                When in doubt, use the default repository level registration.
+              </li>
+            </ul>
+            <div class="form-check">
+              <input
+                class="form-check-input"
+                type="radio"
+                bind:group={runnerLevel}
+                value="repo"
+                id="repo"
+              />
+              <label class="form-check-label" for="repo">Repository</label>
+            </div>
+            <div class="form-check">
+              <input
+                class="form-check-input"
+                type="radio"
+                bind:group={runnerLevel}
+                value="org"
+                id="org"
+              />
+              <label class="form-check-label" for="org">Organization</label>
+            </div>
+          </div>
+        {/if}
+
         <h2>Finish Setup</h2>
         <div class="px-3 py-3">
           {#if result === undefined}
@@ -277,7 +382,7 @@
               implications before continuing.</p>
           {/if}
           <button type="submit" class="btn btn-success"
-                  disabled={isSubmitDisabled(instance, auth, existingAppId, existingAppPk, pat, success)}>
+                  disabled={isSubmitDisabled(instance, auth, existingAppId, existingAppPk, runnerLevel, pat, success)}>
             {submitText(auth)}
           </button>
         </div>

diff --git a/src/delete-failed-runner.lambda.ts b/src/delete-failed-runner.lambda.ts
@@ -1,5 +1,5 @@
 import { RequestError } from '@octokit/request-error';
-import { getOctokit, getRunner } from './lambda-github';
+import { deleteRunner, getOctokit, getRunner } from './lambda-github';
 import { StepFunctionLambdaInput } from './lambda-helpers';
 
 class RunnerBusy extends Error {
@@ -20,10 +20,10 @@ class ReraisedError extends Error {
 }
 
 export async function handler(event: StepFunctionLambdaInput) {
-  const { octokit } = await getOctokit(event.installationId);
+  const { octokit, githubSecrets } = await getOctokit(event.installationId);
 
   // find runner id
-  const runner = await getRunner(octokit, event.owner, event.repo, event.runnerName);
+  const runner = await getRunner(octokit, githubSecrets.runnerLevel, event.owner, event.repo, event.runnerName);
   if (!runner) {
     console.error(`Unable to find runner id for ${event.owner}/${event.repo}:${event.runnerName}`);
     throw new ReraisedError(event);
@@ -36,11 +36,7 @@ export async function handler(event: StepFunctionLambdaInput) {
   // we try removing it anyway for cases where a job wasn't accepted, and just in case it wasn't removed.
   // repos have a limited number of self-hosted runners, so we can't leave dead ones behind.
   try {
-    await octokit.rest.actions.deleteSelfHostedRunnerFromRepo({
-      owner: event.owner,
-      repo: event.repo,
-      runner_id: runner.id,
-    });
+    await deleteRunner(octokit, githubSecrets.runnerLevel, event.owner, event.repo, runner.id);
   } catch (e) {
     const reqError = <RequestError>e;
     if (reqError.message.includes('is still running a job')) {

diff --git a/src/idle-runner-repear.lambda.ts b/src/idle-runner-repear.lambda.ts
@@ -1,7 +1,7 @@
 import { DescribeExecutionCommand, SFNClient, StopExecutionCommand } from '@aws-sdk/client-sfn';
 import { Octokit } from '@octokit/rest';
 import * as AWSLambda from 'aws-lambda';
-import { getOctokit, getRunner } from './lambda-github';
+import { deleteRunner, getOctokit, getRunner } from './lambda-github';
 
 interface IdleReaperLambdaInput {
   readonly executionArn: string;
@@ -16,7 +16,8 @@ const sfn = new SFNClient();
 
 export async function handler(event: AWSLambda.SQSEvent): Promise<AWSLambda.SQSBatchResponse> {
   const result: AWSLambda.SQSBatchResponse = { batchItemFailures: [] };
-  const octokitCache: { [key: number]: Octokit } = {};
+  let octokitCache: Octokit | undefined;
+  let runnerLevel: 'repo' | 'org' | undefined;
 
   for (const record of event.Records) {
     const input = JSON.parse(record.body) as IdleReaperLambdaInput;
@@ -33,15 +34,16 @@ export async function handler(event: AWSLambda.SQSEvent): Promise<AWSLambda.SQSB
     }
 
     // get github access
-    let octokit: Octokit;
-    if (octokitCache[input.installationId ?? -1]) {
-      octokit = octokitCache[input.installationId ?? -1];
-    } else {
-      octokit = octokitCache[input.installationId ?? -1] = (await getOctokit(input.installationId)).octokit;
+    if (!octokitCache) {
+      // getOctokit calls secrets manager every time, so cache the result
+      const { octokit, githubSecrets } = await getOctokit(input.installationId);
+      // TODO if installationId changes during normal operations, we may have some records with good installationId, and some with bad
+      octokitCache = octokit;
+      runnerLevel = githubSecrets.runnerLevel;
     }
 
     // find runner
-    const runner = await getRunner(octokit, input.owner, input.repo, input.runnerName);
+    const runner = await getRunner(octokitCache, runnerLevel, input.owner, input.repo, input.runnerName);
     if (!runner) {
       console.error(`Runner not running yet for ${input.owner}/${input.repo}:${input.runnerName}`);
       retryLater();
@@ -89,11 +91,7 @@ export async function handler(event: AWSLambda.SQSEvent): Promise<AWSLambda.SQSB
 
           try {
             console.log(`Deleting runner ${runner.id}...`);
-            await octokit.rest.actions.deleteSelfHostedRunnerFromRepo({
-              owner: input.owner,
-              repo: input.repo,
-              runner_id: runner.id,
-            });
+            await deleteRunner(octokitCache, runnerLevel, input.owner, input.repo, runner.id);
           } catch (e) {
             console.error(`Failed to delete runner ${runner.id}: ${e}`);
             retryLater();