Add new backend system support to AWS catalog provider

.changeset/tricky-hairs-double.md

@@ -0,0 +1,5 @@
+---
+'@roadiehq/catalog-backend-module-aws': minor
+---
+
+Added support for new backend system and full support of `@backtage/integration-aws-node` for session management

package.json

@@ -57,5 +57,6 @@
   "prettier": "@spotify/prettier-config",
   "dependencies": {
     "@changesets/cli": "^2.18.0"
-  }
+  },
+  "packageManager": "[email protected]+sha1.4ba7fc5c6e704fce2066ecbfb0b0d8976fe62447"
 }

plugins/backend/catalog-backend-module-aws/README.md

@@ -3,15 +3,41 @@
 This is an extension module to the plugin-catalog-backend plugin, providing
 entity providers to read AWS objects as Backstage Entities.
 
-You will need to configure the providers in your catalog.ts file in your backstage backend:
+## Installation
+
+Add the module package as backend dependency:
+
+```bash
+# From your Backstage root directory
+yarn --cwd packages/backend add @roadiehq/catalog-backend-module-aws
+```
+
+### New backend system
+
+This backend plugin supports the [new backend system](https://backstage.io/docs/backend-system/), here's how you can set that up:
+
+In your `packages/backend/src/index.ts`, Add the collator to your backend instance, along with the search plugin itself:
+
+```diff
+import { createBackend } from '@backstage/backend-defaults';
+
+const backend = createBackend();
+backend.add(import('@backstage/plugin-catalog-backend/alpha'));
++ backend.add(import('@roadiehq/catalog-backend-module-aws/alpha'));
+backend.start();
+```
+
+### Legacy backend system
+
+Configure the providers in your catalog.ts file in your backstage backend:
 
 ```typescript
 import {
   AWSLambdaFunctionProvider,
   AWSS3BucketProvider,
   AWSIAMUserProvider,
   AWSEC2Provider,
-} from '@roadiehq/catalog-backend-module-aws';
+} from '@roadihq/catalog-backend-module-aws';
 
 export default async function createPlugin(
   env: PluginEnvironment,
@@ -47,6 +73,33 @@ export default async function createPlugin(
 }
 ```
 
+## Configuration
+
+The plugin is coupled to the [@backstage/integration-aws-node](https://github.com/backstage/backstage/tree/master/packages/integration-aws-node) plugin. It only adds the `aws.accounts[].schedule` allowing to configure different collection periods on per account basis.
+
+```yaml
+aws:
+  accounts:
+    - accountId: '123456789012' # Must have quotes to avoid bad interpretation as integer
+    - accountId: '210987654321'
+      schedule:
+        frequency: { hours: 2 }
+        timeout: { seconds: 45 }
+        initialDelay: { seconds: 15 }
+```
+
+Similarly, the AWS Organization configuration uses the same specification as [@backstage/plugin-catalog-backend-module-aws](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-aws/README.md) plugin.
+
+```yaml
+catalog:
+  processors:
+    awsOragnization:
+      provider:
+        accountId: '123456789012'
+```
+
+Checkout the [config.d.ts](./config.d.ts) for more options.
+
 ---
 
 Roadie gives you a hassle-free, fully customisable SaaS Backstage. Find out more here: [https://roadie.io](https://roadie.io).

plugins/backend/catalog-backend-module-aws/config.d.ts

@@ -13,13 +13,84 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { AWSAccountProviderConfig } from './src/types';
+
+import { SchedulerServiceTaskScheduleDefinitionConfig } from '@backstage/backend-plugin-api';
 
 export interface Config {
-  integrations?: {
+  catalog?: {
+    processors?: {
+      /**
+       * AwsOrganizationCloudAccountProcessor and AWSOrganizationAccountsProvider configuration
+       */
+      awsOrganization?: {
+        provider: {
+          /**
+           * The role to be assumed by this processor
+           * @deprecated Use `accountId` instead.
+           */
+          roleArn?: string;
+
+          /**
+           * The AWS account ID to query for organizational data
+           */
+          accountId?: string;
+        };
+      };
+    };
+  };
+  /** Configuration for access to AWS accounts */
+  aws?: {
     /**
-     * AWS configuration
+     * Configuration for retrieving AWS accounts credentials
      */
-    aws?: AWSAccountProviderConfig[];
+    accounts?: Array<{
+      /**
+       * The account ID of the target account that this matches on, e.g. "123456789012"
+       */
+      accountId: string;
+
+      /**
+       * The access key ID for a set of static AWS credentials
+       * @visibility secret
+       */
+      accessKeyId?: string;
+
+      /**
+       * The secret access key for a set of static AWS credentials
+       * @visibility secret
+       */
+      secretAccessKey?: string;
+
+      /**
+       * The configuration profile from a credentials file at ~/.aws/credentials and
+       * a configuration file at ~/.aws/config.
+       */
+      profile?: string;
+
+      /**
+       * The IAM role to assume to retrieve temporary AWS credentials
+       */
+      roleName?: string;
+
+      /**
+       * The AWS partition of the IAM role, e.g. "aws", "aws-cn"
+       */
+      partition?: string;
+
+      /**
+       * The STS regional endpoint to use when retrieving temporary AWS credentials, e.g. "ap-northeast-1"
+       */
+      region?: string;
+
+      /**
+       * The unique identifier needed to assume the role to retrieve temporary AWS credentials
+       * @visibility secret
+       */
+      externalId?: string;
+      /**
+       * (Optional) TaskScheduleDefinition for the refresh.
+       */
+      schedule?: SchedulerServiceTaskScheduleDefinitionConfig;
+    }>;
   };
 }

plugins/backend/catalog-backend-module-aws/package.json

@@ -51,23 +51,26 @@
     "@aws-sdk/client-sts": "^3.76.0",
     "@aws-sdk/credential-providers": "^3.76.0",
     "@aws-sdk/lib-dynamodb": "^3.76.0",
-    "@backstage/integration-aws-node": "^0.1.12",
+    "@aws-sdk/util-arn-parser": "^3.76.0",
     "@backstage/backend-common": "^0.25.0",
+    "@backstage/backend-plugin-api": "^1.0.1",
     "@backstage/catalog-client": "^1.7.1",
-    "@backstage/plugin-kubernetes-common": "^0.8.3",
     "@backstage/catalog-model": "^1.7.0",
     "@backstage/config": "^1.2.0",
     "@backstage/errors": "^1.2.4",
+    "@backstage/integration-aws-node": "^0.1.12",
     "@backstage/plugin-catalog-backend": "^1.27.0",
+    "@backstage/plugin-catalog-common": "^1.1.0",
     "@backstage/plugin-catalog-node": "^1.13.1",
+    "@backstage/plugin-kubernetes-common": "^0.8.3",
     "@backstage/types": "^1.1.1",
-    "@aws-sdk/util-arn-parser": "^3.76.0",
     "link2aws": "^1.0.18",
     "lodash": "^4.17.21",
     "p-limit": "^3.0.2",
     "winston": "^3.2.1"
   },
   "devDependencies": {
+    "@backstage/backend-test-utils": "^1.0.1",
     "@backstage/cli": "^0.28.0",
     "@testing-library/jest-dom": "^6.4.2",
     "@types/link2aws": "^1.0.0",

plugins/backend/catalog-backend-module-aws/src/alpha.ts

@@ -0,0 +1,126 @@
+/*
+ * Copyright 2024 Larder Software Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  coreServices,
+  createBackendModule,
+} from '@backstage/backend-plugin-api';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import {
+  AWSLambdaFunctionProvider,
+  AWSS3BucketProvider,
+  AWSIAMUserProvider,
+  AWSDynamoDbTableProvider,
+  // AWSDynamoDbTableDataProvider,
+  AWSIAMRoleProvider,
+  AWSEKSClusterProvider,
+  AWSEC2Provider,
+  AWSRDSProvider,
+  AWSOrganizationAccountsProvider,
+} from './providers';
+import { AWSIAMRoleProcessor } from './processors';
+
+export default createBackendModule({
+  pluginId: 'catalog',
+  moduleId: 'aws-entity-provider',
+  register(env) {
+    env.registerInit({
+      deps: {
+        config: coreServices.rootConfig,
+        logger: coreServices.logger,
+        scheduler: coreServices.scheduler,
+        catalog: catalogProcessingExtensionPoint,
+        discovery: coreServices.discovery,
+      },
+      async init({ config, logger, scheduler, catalog, discovery }) {
+        if (
+          config.getOptional(
+            'catalog.processors.awsOrganization.provider.accountId',
+          )
+        ) {
+          catalog.addEntityProvider(
+            AWSOrganizationAccountsProvider.fromConfig(
+              config.get('catalog.processors.awsOrganization.provider'),
+              { logger, scheduler },
+            ),
+          );
+        }
+
+        for (const awsConfig of config.getOptionalConfigArray('aws.accounts') ||
+          []) {
+          // Amazon DynamoDB table data
+          // await catalog.addEntityProvider(
+          //   AWSDynamoDbTableDataProvider.fromConfig(awsConfig, {
+          //     logger,
+          //     scheduler,
+          //   }),
+          // );
+
+          // Amazon DynamoDB table
+          await catalog.addEntityProvider(
+            AWSDynamoDbTableProvider.fromConfig(awsConfig, {
+              logger,
+              scheduler,
+            }),
+          );
+
+          // Amazon EKS
+          await catalog.addEntityProvider(
+            AWSEKSClusterProvider.fromConfig(awsConfig, { logger, scheduler }),
+          );
+
+          // Amazon EC2
+          await catalog.addEntityProvider(
+            AWSEC2Provider.fromConfig(awsConfig, { logger, scheduler }),
+          );
+
+          // AWS Identity and Access Management role
+          await catalog.addEntityProvider(
+            AWSIAMRoleProvider.fromConfig(awsConfig, { logger, scheduler }),
+          );
+
+          // AWS Identity and Access Management user
+          await catalog.addEntityProvider(
+            AWSIAMUserProvider.fromConfig(awsConfig, { logger, scheduler }),
+          );
+
+          // AWS Lambda function
+          await catalog.addEntityProvider(
+            AWSLambdaFunctionProvider.fromConfig(awsConfig, {
+              logger,
+              scheduler,
+            }),
+          );
+
+          // Amazon Relational Database Service (RDS)
+          await catalog.addEntityProvider(
+            AWSRDSProvider.fromConfig(awsConfig, { logger, scheduler }),
+          );
+
+          // Amazon Simple Storage Service (S3)
+          await catalog.addEntityProvider(
+            AWSS3BucketProvider.fromConfig(awsConfig, { logger, scheduler }),
+          );
+
+          await catalog.addProcessor(
+            AWSIAMRoleProcessor.fromConfig(awsConfig, { logger, discovery }),
+          );
+        }
+
+        logger.info('AWS entity providers registered');
+      },
+    });
+  },
+});

plugins/backend/catalog-backend-module-aws/src/processors/AWSCatalogProcessor.ts

@@ -14,20 +14,20 @@
  * limitations under the License.
  */
 
-import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
+import { CatalogProcessor } from '@backstage/plugin-catalog-node';
 import { CatalogApi } from '@backstage/catalog-client';
-import { Logger } from 'winston';
+import { LoggerService } from '@backstage/backend-plugin-api';
 
 export abstract class AWSCatalogProcessor implements CatalogProcessor {
   protected readonly catalogApi: CatalogApi;
-  protected readonly logger: Logger;
+  protected readonly logger: LoggerService;
   public abstract getProcessorName(): string;
   constructor({
     catalogApi,
     logger,
   }: {
     catalogApi: CatalogApi;
-    logger: Logger;
+    logger: LoggerService;
   }) {
     this.catalogApi = catalogApi;
     this.logger = logger;

plugins/backend/catalog-backend-module-aws/src/processors/AWSIAMRoleProcessor.ts

@@ -24,20 +24,22 @@ import {
 } from '@backstage/catalog-model';
 import {
   CatalogProcessorEmit,
-  LocationSpec,
   processingResult,
-} from '@backstage/plugin-catalog-backend';
+} from '@backstage/plugin-catalog-node';
+
+import { LocationSpec } from '@backstage/plugin-catalog-common';
+
 import { ANNOTATION_AWS_IAM_ROLE_ARN } from '../annotations';
 import { Config } from '@backstage/config';
-import * as winston from 'winston';
-import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { LoggerService } from '@backstage/backend-plugin-api';
+import { DiscoveryService } from '@backstage/backend-plugin-api';
 import { CatalogClient, CatalogApi } from '@backstage/catalog-client';
 import { arnToName } from '../utils/arnToName';
 
 export class AWSIAMRoleProcessor extends AWSCatalogProcessor {
   static fromConfig(
     _config: Config,
-    options: { logger: winston.Logger; discovery: PluginEndpointDiscovery },
+    options: { logger: LoggerService; discovery: DiscoveryService },
   ) {
     const catalogApi: CatalogApi = new CatalogClient({
       discoveryApi: options.discovery,