From 20961ebf15a12e5445f64a9a215de0aa98b598ac Mon Sep 17 00:00:00 2001 From: Florian JUDITH Date: Fri, 1 Nov 2024 01:33:48 -0400 Subject: [PATCH 1/6] Fixed support for the new backend system - Updated installation instructions - Aligned configuration options with `@backstage/catalog-backend-module-aws` - Removed deprecated roleArn - Added pre account scheduling support - Updated tests to leverage Mockservices --- package.json | 3 +- .../catalog-backend-module-aws/README.md | 57 +++++++- .../catalog-backend-module-aws/config.d.ts | 79 ++++++++++- .../catalog-backend-module-aws/package.json | 9 +- .../catalog-backend-module-aws/src/alpha.ts | 126 +++++++++++++++++ .../src/processors/AWSCatalogProcessor.ts | 8 +- .../src/processors/AWSIAMRoleProcessor.ts | 12 +- .../AWSDynamoDbTableDataProvider.test.ts | 8 +- .../providers/AWSDynamoDbTableDataProvider.ts | 102 ++++++++++---- .../src/providers/AWSDynamoDbTableProvider.ts | 116 +++++++++++----- .../src/providers/AWSEC2Provider.ts | 119 +++++++++++----- .../providers/AWSEKSClusterProvider.test.ts | 30 ++-- .../src/providers/AWSEKSClusterProvider.ts | 131 ++++++++++++------ .../src/providers/AWSEntityProvider.ts | 104 +++++--------- .../src/providers/AWSIAMRoleProvider.test.ts | 20 ++- .../src/providers/AWSIAMRoleProvider.ts | 118 +++++++++++----- .../src/providers/AWSIAMUserProvider.test.ts | 19 ++- .../src/providers/AWSIAMUserProvider.ts | 116 +++++++++++----- .../AWSLambdaFunctionProvider.test.ts | 67 ++------- .../providers/AWSLambdaFunctionProvider.ts | 122 +++++++++++----- .../AWSOrganizationAccountsProvider.ts | 124 +++++++++++------ .../src/providers/AWSRDSProvider.ts | 118 +++++++++++----- .../src/providers/AWSS3BucketProvider.test.ts | 19 ++- .../src/providers/AWSS3BucketProvider.ts | 118 +++++++++++----- .../catalog-backend-module-aws/src/types.ts | 8 +- yarn.lock | 43 ++---- 26 files changed, 1220 insertions(+), 576 deletions(-) create mode 100644 plugins/backend/catalog-backend-module-aws/src/alpha.ts diff --git a/package.json b/package.json index 0eb1b7776..8c19a2bf1 100644 --- a/package.json +++ b/package.json @@ -57,5 +57,6 @@ "prettier": "@spotify/prettier-config", "dependencies": { "@changesets/cli": "^2.18.0" - } + }, + "packageManager": "yarn@1.22.19+sha1.4ba7fc5c6e704fce2066ecbfb0b0d8976fe62447" } diff --git a/plugins/backend/catalog-backend-module-aws/README.md b/plugins/backend/catalog-backend-module-aws/README.md index 2aa6466bd..ca25ad844 100644 --- a/plugins/backend/catalog-backend-module-aws/README.md +++ b/plugins/backend/catalog-backend-module-aws/README.md @@ -3,7 +3,33 @@ This is an extension module to the plugin-catalog-backend plugin, providing entity providers to read AWS objects as Backstage Entities. -You will need to configure the providers in your catalog.ts file in your backstage backend: +## Installation + +Add the module package as backend dependency: + +```bash +# From your Backstage root directory +yarn --cwd packages/backend add @roadiehq/catalog-backend-module-aws +``` + +### New backend system + +This backend plugin supports the [new backend system](https://backstage.io/docs/backend-system/), here's how you can set that up: + +In your `packages/backend/src/index.ts`, Add the collator to your backend instance, along with the search plugin itself: + +```diff +import { createBackend } from '@backstage/backend-defaults'; + +const backend = createBackend(); +backend.add(import('@backstage/plugin-catalog-backend/alpha')); ++ backend.add(import('@roadiehq/catalog-backend-module-aws/alpha')); +backend.start(); +``` + +### Legacy backend system + +Configure the providers in your catalog.ts file in your backstage backend: ```typescript import { @@ -11,7 +37,7 @@ import { AWSS3BucketProvider, AWSIAMUserProvider, AWSEC2Provider, -} from '@roadiehq/catalog-backend-module-aws'; +} from '@internal/catalog-backend-module-aws'; export default async function createPlugin( env: PluginEnvironment, @@ -47,6 +73,33 @@ export default async function createPlugin( } ``` +## Configuration + +The plugin is coupled to the [@backstage/integration-aws-node](https://github.com/backstage/backstage/tree/master/packages/integration-aws-node) plugin. It only adds the `aws.accounts[].schedule` allowing to configure different collection periods on per account basis. + +```yaml +aws: + accounts: + - accountId: '123456789012' # Must have quotes to avoid bad interpretation as integer + - accountId: '210987654321' + schedule: + frequency: { hours: 2 } + timeout: { seconds: 45 } + initialDelay: { seconds: 15 } +``` + +Similarly, the AWS Organization configuration uses the same specification as [@backstage/plugin-catalog-backend-module-aws](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-aws/README.md) plugin. + +```yaml +catalog: + processors: + awsOragnization: + provider: + accountId: '123456789012' +``` + +Checkout the [config.d.ts](./config.d.ts) for more options. + --- Roadie gives you a hassle-free, fully customisable SaaS Backstage. Find out more here: [https://roadie.io](https://roadie.io). diff --git a/plugins/backend/catalog-backend-module-aws/config.d.ts b/plugins/backend/catalog-backend-module-aws/config.d.ts index 72cacb71e..b665ba194 100644 --- a/plugins/backend/catalog-backend-module-aws/config.d.ts +++ b/plugins/backend/catalog-backend-module-aws/config.d.ts @@ -13,13 +13,84 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { AWSAccountProviderConfig } from './src/types'; + +import { SchedulerServiceTaskScheduleDefinitionConfig } from '@backstage/backend-plugin-api'; export interface Config { - integrations?: { + catalog?: { + processors?: { + /** + * AwsOrganizationCloudAccountProcessor and AWSOrganizationAccountsProvider configuration + */ + awsOrganization?: { + provider: { + /** + * The role to be assumed by this processor + * @deprecated Use `accountId` instead. + */ + roleArn?: string; + + /** + * The AWS account ID to query for organizational data + */ + accountId?: string; + }; + }; + }; + }; + /** Configuration for access to AWS accounts */ + aws?: { /** - * AWS configuration + * Configuration for retrieving AWS accounts credentials */ - aws?: AWSAccountProviderConfig[]; + accounts?: Array<{ + /** + * The account ID of the target account that this matches on, e.g. "123456789012" + */ + accountId: string; + + /** + * The access key ID for a set of static AWS credentials + * @visibility secret + */ + accessKeyId?: string; + + /** + * The secret access key for a set of static AWS credentials + * @visibility secret + */ + secretAccessKey?: string; + + /** + * The configuration profile from a credentials file at ~/.aws/credentials and + * a configuration file at ~/.aws/config. + */ + profile?: string; + + /** + * The IAM role to assume to retrieve temporary AWS credentials + */ + roleName?: string; + + /** + * The AWS partition of the IAM role, e.g. "aws", "aws-cn" + */ + partition?: string; + + /** + * The STS regional endpoint to use when retrieving temporary AWS credentials, e.g. "ap-northeast-1" + */ + region?: string; + + /** + * The unique identifier needed to assume the role to retrieve temporary AWS credentials + * @visibility secret + */ + externalId?: string; + /** + * (Optional) TaskScheduleDefinition for the refresh. + */ + schedule?: SchedulerServiceTaskScheduleDefinitionConfig; + }>; }; } diff --git a/plugins/backend/catalog-backend-module-aws/package.json b/plugins/backend/catalog-backend-module-aws/package.json index c64be72b9..a89de9b15 100644 --- a/plugins/backend/catalog-backend-module-aws/package.json +++ b/plugins/backend/catalog-backend-module-aws/package.json @@ -51,23 +51,26 @@ "@aws-sdk/client-sts": "^3.76.0", "@aws-sdk/credential-providers": "^3.76.0", "@aws-sdk/lib-dynamodb": "^3.76.0", - "@backstage/integration-aws-node": "^0.1.12", + "@aws-sdk/util-arn-parser": "^3.76.0", "@backstage/backend-common": "^0.25.0", + "@backstage/backend-plugin-api": "^1.0.1", "@backstage/catalog-client": "^1.7.1", - "@backstage/plugin-kubernetes-common": "^0.8.3", "@backstage/catalog-model": "^1.7.0", "@backstage/config": "^1.2.0", "@backstage/errors": "^1.2.4", + "@backstage/integration-aws-node": "^0.1.12", "@backstage/plugin-catalog-backend": "^1.27.0", + "@backstage/plugin-catalog-common": "^1.1.0", "@backstage/plugin-catalog-node": "^1.13.1", + "@backstage/plugin-kubernetes-common": "^0.8.3", "@backstage/types": "^1.1.1", - "@aws-sdk/util-arn-parser": "^3.76.0", "link2aws": "^1.0.18", "lodash": "^4.17.21", "p-limit": "^3.0.2", "winston": "^3.2.1" }, "devDependencies": { + "@backstage/backend-test-utils": "^1.0.1", "@backstage/cli": "^0.28.0", "@testing-library/jest-dom": "^6.4.2", "@types/link2aws": "^1.0.0", diff --git a/plugins/backend/catalog-backend-module-aws/src/alpha.ts b/plugins/backend/catalog-backend-module-aws/src/alpha.ts new file mode 100644 index 000000000..baffe2add --- /dev/null +++ b/plugins/backend/catalog-backend-module-aws/src/alpha.ts @@ -0,0 +1,126 @@ +/* + * Copyright 2024 Larder Software Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { + AWSLambdaFunctionProvider, + AWSS3BucketProvider, + AWSIAMUserProvider, + AWSDynamoDbTableProvider, + // AWSDynamoDbTableDataProvider, + AWSIAMRoleProvider, + AWSEKSClusterProvider, + AWSEC2Provider, + AWSRDSProvider, + AWSOrganizationAccountsProvider, +} from './providers'; +import { AWSIAMRoleProcessor } from './processors'; + +export default createBackendModule({ + pluginId: 'catalog', + moduleId: 'aws-entity-provider', + register(env) { + env.registerInit({ + deps: { + config: coreServices.rootConfig, + logger: coreServices.logger, + scheduler: coreServices.scheduler, + catalog: catalogProcessingExtensionPoint, + discovery: coreServices.discovery, + }, + async init({ config, logger, scheduler, catalog, discovery }) { + if ( + config.getOptional( + 'catalog.processors.awsOrganization.provider.accountId', + ) + ) { + catalog.addEntityProvider( + AWSOrganizationAccountsProvider.fromConfig( + config.get('catalog.processors.awsOrganization.provider'), + { logger, scheduler }, + ), + ); + } + + for (const awsConfig of config.getOptionalConfigArray('aws.accounts') || + []) { + // Amazon DynamoDB table data + // await catalog.addEntityProvider( + // AWSDynamoDbTableDataProvider.fromConfig(awsConfig, { + // logger, + // scheduler, + // }), + // ); + + // Amazon DynamoDB table + await catalog.addEntityProvider( + AWSDynamoDbTableProvider.fromConfig(awsConfig, { + logger, + scheduler, + }), + ); + + // Amazon EKS + await catalog.addEntityProvider( + AWSEKSClusterProvider.fromConfig(awsConfig, { logger, scheduler }), + ); + + // Amazon EC2 + await catalog.addEntityProvider( + AWSEC2Provider.fromConfig(awsConfig, { logger, scheduler }), + ); + + // AWS Identity and Access Management role + await catalog.addEntityProvider( + AWSIAMRoleProvider.fromConfig(awsConfig, { logger, scheduler }), + ); + + // AWS Identity and Access Management user + await catalog.addEntityProvider( + AWSIAMUserProvider.fromConfig(awsConfig, { logger, scheduler }), + ); + + // AWS Lambda function + await catalog.addEntityProvider( + AWSLambdaFunctionProvider.fromConfig(awsConfig, { + logger, + scheduler, + }), + ); + + // Amazon Relational Database Service (RDS) + await catalog.addEntityProvider( + AWSRDSProvider.fromConfig(awsConfig, { logger, scheduler }), + ); + + // Amazon Simple Storage Service (S3) + await catalog.addEntityProvider( + AWSS3BucketProvider.fromConfig(awsConfig, { logger, scheduler }), + ); + + await catalog.addProcessor( + AWSIAMRoleProcessor.fromConfig(awsConfig, { logger, discovery }), + ); + } + + logger.info('AWS entity providers registered'); + }, + }); + }, +}); diff --git a/plugins/backend/catalog-backend-module-aws/src/processors/AWSCatalogProcessor.ts b/plugins/backend/catalog-backend-module-aws/src/processors/AWSCatalogProcessor.ts index b96985d7a..f0f29e939 100644 --- a/plugins/backend/catalog-backend-module-aws/src/processors/AWSCatalogProcessor.ts +++ b/plugins/backend/catalog-backend-module-aws/src/processors/AWSCatalogProcessor.ts @@ -14,20 +14,20 @@ * limitations under the License. */ -import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; +import { CatalogProcessor } from '@backstage/plugin-catalog-node'; import { CatalogApi } from '@backstage/catalog-client'; -import { Logger } from 'winston'; +import { LoggerService } from '@backstage/backend-plugin-api'; export abstract class AWSCatalogProcessor implements CatalogProcessor { protected readonly catalogApi: CatalogApi; - protected readonly logger: Logger; + protected readonly logger: LoggerService; public abstract getProcessorName(): string; constructor({ catalogApi, logger, }: { catalogApi: CatalogApi; - logger: Logger; + logger: LoggerService; }) { this.catalogApi = catalogApi; this.logger = logger; diff --git a/plugins/backend/catalog-backend-module-aws/src/processors/AWSIAMRoleProcessor.ts b/plugins/backend/catalog-backend-module-aws/src/processors/AWSIAMRoleProcessor.ts index 79bec58f0..ef63d35be 100644 --- a/plugins/backend/catalog-backend-module-aws/src/processors/AWSIAMRoleProcessor.ts +++ b/plugins/backend/catalog-backend-module-aws/src/processors/AWSIAMRoleProcessor.ts @@ -24,20 +24,22 @@ import { } from '@backstage/catalog-model'; import { CatalogProcessorEmit, - LocationSpec, processingResult, -} from '@backstage/plugin-catalog-backend'; +} from '@backstage/plugin-catalog-node'; + +import { LocationSpec } from '@backstage/plugin-catalog-common'; + import { ANNOTATION_AWS_IAM_ROLE_ARN } from '../annotations'; import { Config } from '@backstage/config'; -import * as winston from 'winston'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { DiscoveryService } from '@backstage/backend-plugin-api'; import { CatalogClient, CatalogApi } from '@backstage/catalog-client'; import { arnToName } from '../utils/arnToName'; export class AWSIAMRoleProcessor extends AWSCatalogProcessor { static fromConfig( _config: Config, - options: { logger: winston.Logger; discovery: PluginEndpointDiscovery }, + options: { logger: LoggerService; discovery: DiscoveryService }, ) { const catalogApi: CatalogApi = new CatalogClient({ discoveryApi: options.discovery, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.test.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.test.ts index 7ca35d3ab..e905b042e 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.test.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.test.ts @@ -14,9 +14,9 @@ * limitations under the License. */ +import { mockServices } from '@backstage/backend-test-utils'; import { AWSDynamoDbTableDataProvider } from './AWSDynamoDbTableDataProvider'; import { ConfigReader } from '@backstage/config'; -import { getVoidLogger } from '@backstage/backend-common'; const validConfig = { accountId: '123456789012', @@ -48,7 +48,8 @@ describe('AWSDynamoDbTableDataProvider', () => { ]); const testWrapper = () => { AWSDynamoDbTableDataProvider.fromConfig(config, { - logger: getVoidLogger(), + logger: mockServices.logger.mock(), + scheduler: mockServices.scheduler.mock(), }); }; expect(testWrapper).toThrow( @@ -64,7 +65,8 @@ describe('AWSDynamoDbTableDataProvider', () => { ]); const testWrapper = () => { AWSDynamoDbTableDataProvider.fromConfig(config, { - logger: getVoidLogger(), + logger: mockServices.logger.mock(), + scheduler: mockServices.scheduler.mock(), }); }; expect(testWrapper).not.toThrow(); diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.ts index 4e423400e..79c066327 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.ts @@ -13,24 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - import { - ANNOTATION_LOCATION, - ANNOTATION_ORIGIN_LOCATION, -} from '@backstage/catalog-model'; + LoggerService, + SchedulerService, + readSchedulerServiceTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-plugin-api'; import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, +} from '@backstage/catalog-model'; import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; import { STS } from '@aws-sdk/client-sts'; import { Config } from '@backstage/config'; - import { DynamoDB, DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb'; import { merge } from 'lodash'; import { mapColumnsToEntityValues } from '../utils/columnMapper'; -import * as winston from 'winston'; import { ANNOTATION_ACCOUNT_ID, ANNOTATION_AWS_DDB_TABLE_ARN, @@ -48,42 +50,83 @@ type DdbTableDataConfigOptions = { * Provides entities from AWS DynamoDB service. */ export class AWSDynamoDbTableDataProvider implements EntityProvider { + private readonly logger: LoggerService; + private readonly scheduler: SchedulerService; private readonly accountId: string; - private readonly roleArn: string; - private readonly tableDataConfig: DdbTableDataConfigOptions; - private readonly logger: winston.Logger; + private readonly roleArn?: string; + private readonly tableDataConfig?: DdbTableDataConfigOptions; private connection?: EntityProviderConnection; - static fromConfig(config: Config, options: { logger: winston.Logger }) { - return new AWSDynamoDbTableDataProvider( - config.getString('accountId'), - config.getString('roleName'), - config.get('dynamodbTableData'), + static fromConfig( + config: Config, + options: { logger: LoggerService; scheduler: SchedulerService }, + ) { + const p = new AWSDynamoDbTableDataProvider( options.logger, + options.scheduler, + config.getString('accountId'), + config.getOptionalString('roleName'), + config.getOptional('dynamodbTableData'), ); + + const defaultSchedule = { + frequency: { minutes: 120 }, + timeout: { minutes: 60 }, + initialDelay: { seconds: 30 }, + }; + + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : defaultSchedule; + + options.scheduler.scheduleTask({ + frequency: schedule.frequency, + timeout: schedule.timeout, + initialDelay: schedule.initialDelay, + id: 'amazon-dynamodb-table-data-entity-provider', + fn: p.run, + }); + + return p; } + /** [1] */ constructor( + logger: LoggerService, + scheduler: SchedulerService, accountId: string, - roleArn: string, - options: DdbTableDataConfigOptions, - logger: winston.Logger, + roleArn?: string, + options?: DdbTableDataConfigOptions, ) { + this.logger = logger; + this.scheduler = scheduler; this.accountId = accountId; this.roleArn = roleArn; this.tableDataConfig = options; - this.logger = logger; } + /** [2] */ getProviderName(): string { - return `aws-dynamo-db-table-${this.accountId}-${this.tableDataConfig.tableName}`; + return `aws-dynamo-db-table-${this.accountId}-${this.tableDataConfig?.tableName}`; } + /** [3] */ async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); this.connection = connection; + this.scheduler.scheduleTask({ + frequency: { seconds: 5 }, + timeout: { seconds: 30 }, + id: 'sync-mta-catalog', + fn: this.run, + }); + await this.run(); } + /** [4] */ async run(): Promise { if (!this.connection) { throw new Error('Not initialized'); @@ -105,24 +148,25 @@ export class AWSDynamoDbTableDataProvider implements EntityProvider { const defaultAnnotations: { [name: string]: string } = { [ANNOTATION_LOCATION]: `${this.getProviderName()}:${this.roleArn}`, [ANNOTATION_ORIGIN_LOCATION]: `${this.getProviderName()}:${this.roleArn}`, - 'amazon.com/dynamodb-table-name': this.tableDataConfig.tableName, + 'amazon.com/dynamodb-table-name': this.tableDataConfig + ?.tableName as string, }; if (account.Account) { defaultAnnotations[ANNOTATION_ACCOUNT_ID] = account.Account; } - this.logger.info(`Querying table ${this.tableDataConfig.tableName}`); + this.logger.info(`Querying table ${this.tableDataConfig?.tableName}`); try { const tableRows = await ddb.scan({ - TableName: this.tableDataConfig.tableName, + TableName: this.tableDataConfig?.tableName, }); const tableDescription = await dynamoDBClient.describeTable({ - TableName: this.tableDataConfig.tableName, + TableName: this.tableDataConfig?.tableName, }); const idColumn = - this.tableDataConfig.identifierColumn || + this.tableDataConfig?.identifierColumn || tableDescription.Table?.KeySchema?.[0].AttributeName; if (!idColumn) { throw new Error( @@ -133,7 +177,7 @@ export class AWSDynamoDbTableDataProvider implements EntityProvider { const entities = tableRows.Items?.map(row => { const o = { - kind: this.tableDataConfig.entityKind ?? 'Component', + kind: this.tableDataConfig?.entityKind ?? 'Component', apiVersion: 'backstage.io/v1beta1', metadata: { annotations: { @@ -151,9 +195,9 @@ export class AWSDynamoDbTableDataProvider implements EntityProvider { lifecycle: 'production', }, }; - const mappedColumns = this.tableDataConfig.columnValueMapping + const mappedColumns = this.tableDataConfig?.columnValueMapping ? mapColumnsToEntityValues( - this.tableDataConfig.columnValueMapping, + this.tableDataConfig?.columnValueMapping, row, ) : {}; @@ -167,8 +211,8 @@ export class AWSDynamoDbTableDataProvider implements EntityProvider { locationKey: this.getProviderName(), })), }); - } catch (e) { - this.logger.error(e); + } catch (err) { + this.logger.error(`{e}`); } } } diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts index 20e7f1d94..24ba85b7a 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts @@ -14,9 +14,15 @@ * limitations under the License. */ +import { + LoggerService, + SchedulerService, + readSchedulerServiceTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-plugin-api'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { DynamoDB, paginateListTables } from '@aws-sdk/client-dynamodb'; import { Config } from '@backstage/config'; -import * as winston from 'winston'; +import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node'; import { AWSEntityProvider } from './AWSEntityProvider'; import { ResourceEntity } from '@backstage/catalog-model'; import { ANNOTATION_AWS_DDB_TABLE_ARN } from '../annotations'; @@ -27,62 +33,106 @@ import { relationshipsFromTags, } from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; -import { DynamicAccountConfig } from '../types'; import { duration } from '../utils/timer'; +export type AWSDynamoDbTableProviderOptions = { + logger: LoggerService; + scheduler: SchedulerService; + catalogApi?: CatalogApi; + providerId?: string; + ownerTag?: string; + useTemporaryCredentials?: boolean; + labelValueMapper?: LabelValueMapper; +}; + /** * Provides entities from AWS DynamoDB service. */ export class AWSDynamoDbTableProvider extends AWSEntityProvider { + /** [1] */ static fromConfig( config: Config, - options: { - logger: winston.Logger; - catalogApi?: CatalogApi; - providerId?: string; - ownerTag?: string; - useTemporaryCredentials?: boolean; - labelValueMapper?: LabelValueMapper; - }, - ) { - const accountId = config.getString('accountId'); - const roleArn = config.getOptionalString('roleArn'); - const roleName = config.getString('roleName'); - const externalId = config.getOptionalString('externalId'); - const region = config.getString('region'); - - return new AWSDynamoDbTableProvider( - { accountId, roleName, roleArn, externalId, region }, - options, - ); + options: AWSDynamoDbTableProviderOptions, + ): AWSDynamoDbTableProvider { + const p = new AWSDynamoDbTableProvider(config, options); + + const defaultSchedule = { + frequency: { minutes: 120 }, + timeout: { minutes: 60 }, + initialDelay: { seconds: 30 }, + }; + + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : defaultSchedule; + + options.scheduler.scheduleTask({ + frequency: schedule.frequency, + timeout: schedule.timeout, + initialDelay: schedule.initialDelay, + id: 'amazon-s3-bucket-entity-provider', + fn: p.run, + }); + + return p; } + /** [2] */ getProviderName(): string { return `aws-dynamo-db-table-${this.providerId ?? 0}`; } - private async getDdb(dynamicAccountConfig?: DynamicAccountConfig) { - const credentials = this.useTemporaryCredentials - ? this.getCredentials(dynamicAccountConfig) - : await this.getCredentialsProvider(); - return this.useTemporaryCredentials - ? new DynamoDB({ credentials }) - : new DynamoDB(credentials); + /** [3] */ + async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); + this.connection = connection; + this.scheduler.scheduleTask({ + frequency: { seconds: 5 }, + timeout: { seconds: 30 }, + id: 'amazon-s3-bucket-entity-provider', + fn: this.run, + }); + await this.run(); + } + + private async getDdb() { + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig( + this.config, + ); + const awsCredentialProvider = + await awsCredentialsManager.getCredentialProvider({ accountId }); + return new DynamoDB({ + region, + credentialDefaultProvider: () => + awsCredentialProvider.sdkCredentialProvider, + }); } - async run(dynamicAccountConfig?: DynamicAccountConfig): Promise { + /** [4] */ + async run(): Promise { if (!this.connection) { + this.logger.info('Not initialized'); throw new Error('Not initialized'); } + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const startTimestamp = process.hrtime(); - const { accountId } = this.getParsedConfig(dynamicAccountConfig); const groups = await this.getGroups(); + this.logger.info(`Retrieving all DynamoDB tables for account ${accountId}`); + + const ddb = await this.getDdb(); + const defaultAnnotations = await this.buildDefaultAnnotations( - dynamicAccountConfig, + this.config, + accountId, + region, ); - const ddb = await this.getDdb(dynamicAccountConfig); - this.logger.info(`Retrieving all DynamoDB tables for account ${accountId}`); const paginatorConfig = { client: ddb, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts index a2fd4617b..d8d2884b0 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts @@ -14,9 +14,15 @@ * limitations under the License. */ +import { + LoggerService, + SchedulerService, + readSchedulerServiceTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-plugin-api'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { ANNOTATION_VIEW_URL, ResourceEntity } from '@backstage/catalog-model'; import { EC2 } from '@aws-sdk/client-ec2'; -import * as winston from 'winston'; +import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node'; import { Config } from '@backstage/config'; import { AWSEntityProvider } from './AWSEntityProvider'; import { ANNOTATION_AWS_EC2_INSTANCE_ID } from '../annotations'; @@ -27,66 +33,107 @@ import { relationshipsFromTags, } from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; -import { DynamicAccountConfig } from '../types'; import { duration } from '../utils/timer'; +export type AWSEC2ProviderOptions = { + logger: LoggerService; + scheduler: SchedulerService; + catalogApi?: CatalogApi; + providerId?: string; + ownerTag?: string; + useTemporaryCredentials?: boolean; + labelValueMapper?: LabelValueMapper; +}; + /** * Provides entities from AWS Elastic Compute Cloud. */ export class AWSEC2Provider extends AWSEntityProvider { + /** [1] */ static fromConfig( config: Config, - options: { - logger: winston.Logger; - catalogApi?: CatalogApi; - providerId?: string; - ownerTag?: string; - useTemporaryCredentials?: boolean; - labelValueMapper?: LabelValueMapper; - }, - ) { - const accountId = config.getString('accountId'); - const roleName = config.getString('roleName'); - const roleArn = config.getOptionalString('roleArn'); - const externalId = config.getOptionalString('externalId'); - const region = config.getString('region'); - - return new AWSEC2Provider( - { accountId, roleName, roleArn, externalId, region }, - options, - ); + options: AWSEC2ProviderOptions, + ): AWSEC2Provider { + const p = new AWSEC2Provider(config, options); + + const defaultSchedule = { + frequency: { minutes: 120 }, + timeout: { minutes: 60 }, + initialDelay: { seconds: 30 }, + }; + + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : defaultSchedule; + + options.scheduler.scheduleTask({ + frequency: schedule.frequency, + timeout: schedule.timeout, + initialDelay: schedule.initialDelay, + id: 'amazon-ec2-entity-provider', + fn: p.run, + }); + + return p; } + /** [2] */ getProviderName(): string { - return `aws-ec2-provider-${this.providerId ?? 0}`; + return `amazon-ec2-provider-${this.providerId ?? 0}`; } - private async getEc2(dynamicAccountConfig?: DynamicAccountConfig) { - const { region } = this.getParsedConfig(dynamicAccountConfig); - const credentials = this.useTemporaryCredentials - ? this.getCredentials(dynamicAccountConfig) - : await this.getCredentialsProvider(); - return this.useTemporaryCredentials - ? new EC2({ credentials, region: region }) - : new EC2(credentials); + /** [3] */ + async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); + this.connection = connection; + this.scheduler.scheduleTask({ + frequency: { seconds: 5 }, + timeout: { seconds: 30 }, + id: 'amazon-ec2-entity-provider', + fn: this.run, + }); + await this.run(); + } + + private async getEc2() { + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig( + this.config, + ); + const awsCredentialProvider = + await awsCredentialsManager.getCredentialProvider({ accountId }); + return new EC2({ + region, + credentialDefaultProvider: () => + awsCredentialProvider.sdkCredentialProvider, + }); } - async run(dynamicAccountConfig?: DynamicAccountConfig): Promise { + /** [4] */ + async run(): Promise { if (!this.connection) { + this.logger.info('Not initialized'); throw new Error('Not initialized'); } - const startTimestamp = process.hrtime(); + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; - const { region, accountId } = this.getParsedConfig(dynamicAccountConfig); + const startTimestamp = process.hrtime(); const groups = await this.getGroups(); this.logger.info(`Providing ec2 resources from aws: ${accountId}`); const ec2Resources: ResourceEntity[] = []; - const ec2 = await this.getEc2(dynamicAccountConfig); + const ec2 = await this.getEc2(); - const defaultAnnotations = - this.buildDefaultAnnotations(dynamicAccountConfig); + const defaultAnnotations = this.buildDefaultAnnotations( + this.config, + accountId, + region, + ); const instances = await ec2.describeInstances({ Filters: [{ Name: 'instance-state-name', Values: ['running'] }], diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.test.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.test.ts index 73a60e5e0..4956c3f54 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.test.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { mockServices } from '@backstage/backend-test-utils'; import { EKS, ListClustersCommand, @@ -22,9 +23,8 @@ import { import { STS, GetCallerIdentityCommand } from '@aws-sdk/client-sts'; import { mockClient } from 'aws-sdk-client-mock'; -import { createLogger, transports } from 'winston'; import { ConfigReader } from '@backstage/config'; -import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { AWSEKSClusterProvider } from './AWSEKSClusterProvider'; import { ANNOTATION_AWS_EKS_CLUSTER_ARN, @@ -34,9 +34,8 @@ import { const eks = mockClient(EKS); const sts = mockClient(STS); -const logger = createLogger({ - transports: [new transports.Console({ silent: true })], -}); +const logger = mockServices.logger.mock(); +const scheduler = mockServices.scheduler.mock(); describe('AWSEKSClusterProvider', () => { const config = new ConfigReader({ @@ -61,7 +60,10 @@ describe('AWSEKSClusterProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const provider = AWSEKSClusterProvider.fromConfig(config, { logger }); + const provider = AWSEKSClusterProvider.fromConfig(config, { + logger, + scheduler, + }); await provider.connect(entityProviderConnection); await provider.run(); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ @@ -93,7 +95,10 @@ describe('AWSEKSClusterProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const provider = AWSEKSClusterProvider.fromConfig(config, { logger }); + const provider = AWSEKSClusterProvider.fromConfig(config, { + logger, + scheduler, + }); await provider.connect(entityProviderConnection); await provider.run(); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ @@ -106,15 +111,12 @@ describe('AWSEKSClusterProvider', () => { labels: { some_url: 'https---asdfhwef.com-hello-world', }, - name: 'a140791d2b20a847f2c74c62c384f93fb83691d871e80385720bce696a0a05f', - title: '123456789012:eu-west-1:cluster1', + title: 'cluster1', annotations: expect.objectContaining({ [ANNOTATION_AWS_EKS_CLUSTER_ARN]: 'arn:aws:eks:eu-west-1:123456789012:cluster/cluster1', [ANNOTATION_AWS_IAM_ROLE_ARN]: 'arn:aws:iam::123456789012:role/cluster1', - 'kubernetes.io/auth-provider': 'aws', - 'kubernetes.io/x-k8s-aws-id': 'cluster1', }), }), }), @@ -148,6 +150,7 @@ describe('AWSEKSClusterProvider', () => { }; const provider = AWSEKSClusterProvider.fromConfig(config, { logger, + scheduler, labelValueMapper: value => value, }); await provider.connect(entityProviderConnection); @@ -162,15 +165,12 @@ describe('AWSEKSClusterProvider', () => { labels: { some_url: 'https://asdfhwef.com/hello-world', }, - name: 'a140791d2b20a847f2c74c62c384f93fb83691d871e80385720bce696a0a05f', - title: '123456789012:eu-west-1:cluster1', + title: 'cluster1', annotations: expect.objectContaining({ [ANNOTATION_AWS_EKS_CLUSTER_ARN]: 'arn:aws:eks:eu-west-1:123456789012:cluster/cluster1', [ANNOTATION_AWS_IAM_ROLE_ARN]: 'arn:aws:iam::123456789012:role/cluster1', - 'kubernetes.io/auth-provider': 'aws', - 'kubernetes.io/x-k8s-aws-id': 'cluster1', }), }), }), diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts index 8f97342ee..bf19bff6b 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts @@ -14,9 +14,15 @@ * limitations under the License. */ +import { + LoggerService, + SchedulerService, + readSchedulerServiceTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-plugin-api'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { ResourceEntity } from '@backstage/catalog-model'; import { EKS, paginateListClusters } from '@aws-sdk/client-eks'; -import * as winston from 'winston'; +import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node'; import { Config } from '@backstage/config'; import { AWSEntityProvider } from './AWSEntityProvider'; import { @@ -37,43 +43,60 @@ import { relationshipsFromTags, } from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; -import { AccountConfig, DynamicAccountConfig } from '../types'; import { duration } from '../utils/timer'; +export type AWSEKSClusterProviderOptions = { + logger: LoggerService; + scheduler: SchedulerService; + catalogApi?: CatalogApi; + providerId?: string; + ownerTag?: string; + useTemporaryCredentials?: boolean; + labelValueMapper?: LabelValueMapper; + clusterTypeValue?: string; +}; + /** * Provides entities from AWS EKS Cluster service. */ export class AWSEKSClusterProvider extends AWSEntityProvider { private readonly clusterTypeValue: string; + //* * [1] */ static fromConfig( config: Config, - options: { - logger: winston.Logger; - catalogApi?: CatalogApi; - providerId?: string; - ownerTag?: string; - useTemporaryCredentials?: boolean; - labelValueMapper?: LabelValueMapper; - clusterTypeValue?: string; - }, - ) { - const accountId = config.getString('accountId'); - const roleName = config.getString('roleName'); - const roleArn = config.getOptionalString('roleArn'); - const externalId = config.getOptionalString('externalId'); - const region = config.getString('region'); - - return new AWSEKSClusterProvider( - { accountId, roleName, roleArn, externalId, region }, - options, - ); + options: AWSEKSClusterProviderOptions, + ): AWSEKSClusterProvider { + const p = new AWSEKSClusterProvider(config, options); + + const defaultSchedule = { + frequency: { minutes: 120 }, + timeout: { minutes: 60 }, + initialDelay: { seconds: 30 }, + }; + + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : defaultSchedule; + + options.scheduler.scheduleTask({ + frequency: schedule.frequency, + timeout: schedule.timeout, + initialDelay: schedule.initialDelay, + id: 'amazon-eks-cluster-entity-provider', + fn: p.run, + }); + + return p; } constructor( - account: AccountConfig, + config: Config, options: { - logger: winston.Logger; + logger: LoggerService; + scheduler: SchedulerService; catalogApi?: CatalogApi; providerId?: string; ownerTag?: string; @@ -82,40 +105,65 @@ export class AWSEKSClusterProvider extends AWSEntityProvider { clusterTypeValue?: string; }, ) { - super(account, options); + super(config, options); this.clusterTypeValue = options.clusterTypeValue ?? 'eks-cluster'; } + /** [2] */ getProviderName(): string { - return `aws-eks-cluster-${this.providerId ?? 0}`; + return `amazon-eks-cluster-${this.providerId ?? 0}`; } - private async getEks(dynamicAccountConfig?: DynamicAccountConfig) { - const { region } = this.getParsedConfig(dynamicAccountConfig); - const credentials = this.useTemporaryCredentials - ? this.getCredentials(dynamicAccountConfig) - : await this.getCredentialsProvider(); - return this.useTemporaryCredentials - ? new EKS({ credentials, region }) - : new EKS(credentials); + /** [3] */ + async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); + this.connection = connection; + this.scheduler.scheduleTask({ + frequency: { seconds: 5 }, + timeout: { seconds: 30 }, + id: 'amazon-eks-cluster-entity-provider', + fn: this.run, + }); + await this.run(); } - async run(dynamicAccountConfig?: DynamicAccountConfig): Promise { + private async getEks() { + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig( + this.config, + ); + const awsCredentialProvider = + await awsCredentialsManager.getCredentialProvider({ accountId }); + return new EKS({ + region, + credentialDefaultProvider: () => + awsCredentialProvider.sdkCredentialProvider, + }); + } + + /** [4] */ + async run(): Promise { if (!this.connection) { + this.logger.info('Not initialized'); throw new Error('Not initialized'); } + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; const startTimestamp = process.hrtime(); - const { accountId } = this.getParsedConfig(dynamicAccountConfig); const groups = await this.getGroups(); this.logger.info(`Providing EKS cluster resources from AWS: ${accountId}`); const eksResources: ResourceEntity[] = []; - const eks = await this.getEks(dynamicAccountConfig); + const eks = await this.getEks(); - const defaultAnnotations = - this.buildDefaultAnnotations(dynamicAccountConfig); + const defaultAnnotations = this.buildDefaultAnnotations( + this.config, + accountId, + region, + ); const paginatorConfig = { client: eks, @@ -143,11 +191,11 @@ export class AWSEKSClusterProvider extends AWSEntityProvider { } if (cluster.cluster?.arn) { - annotations[ANNOTATION_AWS_EKS_CLUSTER_ARN] = cluster.cluster.arn; + annotations[ANNOTATION_AWS_EKS_CLUSTER_ARN] = cluster.cluster?.arn; } if (cluster.cluster?.roleArn) { - annotations[ANNOTATION_AWS_IAM_ROLE_ARN] = cluster.cluster.roleArn; + annotations[ANNOTATION_AWS_IAM_ROLE_ARN] = cluster.cluster?.roleArn; } if (cluster.cluster?.endpoint) { @@ -165,6 +213,7 @@ export class AWSEKSClusterProvider extends AWSEntityProvider { } annotations[ANNOTATION_KUBERNETES_AUTH_PROVIDER] = 'aws'; + const resource: ResourceEntity = { kind: 'Resource', apiVersion: 'backstage.io/v1beta1', diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEntityProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEntityProvider.ts index 19f23e797..d5b03fd9d 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEntityProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEntityProvider.ts @@ -18,8 +18,7 @@ import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; -import * as winston from 'winston'; -import { AccountConfig, DynamicAccountConfig } from '../types'; +import { LoggerService, SchedulerService } from '@backstage/backend-plugin-api'; import { STS } from '@aws-sdk/client-sts'; import { ANNOTATION_ORIGIN_LOCATION, @@ -27,57 +26,59 @@ import { } from '@backstage/catalog-model'; import { ANNOTATION_ACCOUNT_ID } from '../annotations'; import { CatalogApi } from '@backstage/catalog-client'; +import { Config } from '@backstage/config'; import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node'; -import { ConfigReader } from '@backstage/config'; -import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; -import { parse as parseArn } from '@aws-sdk/util-arn-parser'; import { LabelValueMapper, labelsFromTags, Tag } from '../utils/tags'; export abstract class AWSEntityProvider implements EntityProvider { - protected readonly useTemporaryCredentials: boolean; + protected readonly config: Config; + protected readonly logger: LoggerService; + protected readonly scheduler: SchedulerService; protected readonly providerId?: string; - protected readonly logger: winston.Logger; + protected connection?: EntityProviderConnection; private readonly ownerTag: string | undefined; protected readonly catalogApi?: CatalogApi; - private credentialsManager: DefaultAwsCredentialsManager; - private account: AccountConfig; protected readonly labelValueMapper: LabelValueMapper | undefined; public abstract getProviderName(): string; public abstract run( - dynamicAccountConfig?: DynamicAccountConfig, + config: Config, + accountId: string, + region: string, ): Promise; protected constructor( - account: AccountConfig, + config: Config, options: { - logger: winston.Logger; + logger: LoggerService; + scheduler: SchedulerService; catalogApi?: CatalogApi; providerId?: string; ownerTag?: string; - useTemporaryCredentials?: boolean; labelValueMapper?: LabelValueMapper; }, ) { + this.config = config; this.logger = options.logger; + this.scheduler = options.scheduler; this.providerId = options.providerId; this.ownerTag = options.ownerTag; this.catalogApi = options.catalogApi; - this.account = account; - this.useTemporaryCredentials = !!options.useTemporaryCredentials; - this.credentialsManager = DefaultAwsCredentialsManager.fromConfig( - new ConfigReader({ aws: { accounts: [account] } }), - ); this.labelValueMapper = options.labelValueMapper; } + async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); + this.connection = connection; + } + get accountId() { - return this.account.accountId; + return this.config.getOptionalString('accountId'); } get region() { - return this.account.region; + return this.config.getOptionalString('region') ?? 'us-east-1'; } protected getOwnerTag() { @@ -88,41 +89,6 @@ export abstract class AWSEntityProvider implements EntityProvider { return labelsFromTags(tags, this.labelValueMapper); } - protected getCredentials(dynamicAccountConfig?: DynamicAccountConfig) { - const { roleArn, externalId, region } = - this.getParsedConfig(dynamicAccountConfig); - return fromTemporaryCredentials({ - params: { - RoleArn: roleArn, - ExternalId: externalId, - }, - clientConfig: region ? { region: region } : undefined, - }); - } - - protected getParsedConfig(dynamicAccountConfig?: DynamicAccountConfig) { - const { roleArn, externalId, region } = dynamicAccountConfig - ? dynamicAccountConfig - : { roleArn: undefined, externalId: undefined, region: undefined }; - - const arn = roleArn ?? this.account.roleArn ?? this.account.roleName; - const arnParse = parseArn(arn); - return { - accountId: arnParse?.accountId, - region: region ?? this.region ?? arnParse.region, - externalId: externalId ?? this.account.externalId, - roleArn: arn, - }; - } - - protected async getCredentialsProvider() { - const awsCredentialProvider = - await this.credentialsManager.getCredentialProvider({ - accountId: this.accountId, - }); - return awsCredentialProvider.sdkCredentialProvider; - } - protected async getGroups() { let groups = undefined; if (this.catalogApi) { @@ -139,26 +105,26 @@ export abstract class AWSEntityProvider implements EntityProvider { return groups; } - public async connect(connection: EntityProviderConnection): Promise { - this.connection = connection; - } - protected async buildDefaultAnnotations( - dynamicAccountConfig?: DynamicAccountConfig, + config: Config, + accountId: string, + region: string, ) { - const { region, roleArn } = this.getParsedConfig(dynamicAccountConfig); - const credentials = this.useTemporaryCredentials - ? this.getCredentials(dynamicAccountConfig) - : await this.getCredentialsProvider(); - const sts = this.useTemporaryCredentials - ? new STS({ credentials: credentials, region: region }) - : new STS(credentials); + const awsCredentialsManager = + DefaultAwsCredentialsManager.fromConfig(config); + const awsCredentialProvider = + await awsCredentialsManager.getCredentialProvider({ accountId }); + const sts = new STS({ + region, + credentialDefaultProvider: () => + awsCredentialProvider.sdkCredentialProvider, + }); const account = await sts.getCallerIdentity({}); const defaultAnnotations: { [name: string]: string } = { - [ANNOTATION_LOCATION]: `${this.getProviderName()}:${roleArn}`, - [ANNOTATION_ORIGIN_LOCATION]: `${this.getProviderName()}:${roleArn}`, + [ANNOTATION_LOCATION]: account.Arn as string, + [ANNOTATION_ORIGIN_LOCATION]: account.Arn as string, }; if (account.Account) { diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.test.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.test.ts index 2d3aa1235..13572dfb1 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.test.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.test.ts @@ -14,21 +14,21 @@ * limitations under the License. */ +import { mockServices } from '@backstage/backend-test-utils'; import { IAM, ListRolesCommand, Role } from '@aws-sdk/client-iam'; import { STS, GetCallerIdentityCommand } from '@aws-sdk/client-sts'; import { mockClient } from 'aws-sdk-client-mock'; -import { createLogger, transports } from 'winston'; import { AWSIAMRoleProvider } from './AWSIAMRoleProvider'; import { ConfigReader } from '@backstage/config'; -import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +// const iam = mockClient(IAM); const iam = mockClient(IAM); const sts = mockClient(STS); -const logger = createLogger({ - transports: [new transports.Console({ silent: true })], -}); +const logger = mockServices.logger.mock(); +const scheduler = mockServices.scheduler.mock(); describe('AWSIAMRoleProvider', () => { const config = new ConfigReader({ @@ -53,7 +53,10 @@ describe('AWSIAMRoleProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const provider = AWSIAMRoleProvider.fromConfig(config, { logger }); + const provider = AWSIAMRoleProvider.fromConfig(config, { + logger, + scheduler, + }); provider.connect(entityProviderConnection); await provider.run(); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ @@ -81,7 +84,10 @@ describe('AWSIAMRoleProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const provider = AWSIAMRoleProvider.fromConfig(config, { logger }); + const provider = AWSIAMRoleProvider.fromConfig(config, { + logger, + scheduler, + }); provider.connect(entityProviderConnection); await provider.run(); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts index 79a4b11d8..56d47428f 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts @@ -14,9 +14,15 @@ * limitations under the License. */ +import { + LoggerService, + SchedulerService, + readSchedulerServiceTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-plugin-api'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { ANNOTATION_VIEW_URL, ResourceEntity } from '@backstage/catalog-model'; import { IAM, paginateListRoles } from '@aws-sdk/client-iam'; -import * as winston from 'winston'; +import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node'; import { Config } from '@backstage/config'; import { AWSEntityProvider } from './AWSEntityProvider'; import { ANNOTATION_AWS_IAM_ROLE_ARN } from '../annotations'; @@ -28,66 +34,108 @@ import { relationshipsFromTags, } from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; -import { DynamicAccountConfig } from '../types'; import { duration } from '../utils/timer'; +export type AWSIAMRoleProviderOptions = { + logger: LoggerService; + scheduler: SchedulerService; + catalogApi?: CatalogApi; + providerId?: string; + ownerTag?: string; + useTemporaryCredentials?: boolean; + labelValueMapper?: LabelValueMapper; +}; + /** * Provides entities from AWS IAM Role service. */ export class AWSIAMRoleProvider extends AWSEntityProvider { + declare connection?: EntityProviderConnection; + + /** [1] */ static fromConfig( config: Config, - options: { - logger: winston.Logger; - catalogApi?: CatalogApi; - providerId?: string; - ownerTag?: string; - useTemporaryCredentials?: boolean; - labelValueMapper?: LabelValueMapper; - }, - ) { - const accountId = config.getString('accountId'); - const roleName = config.getString('roleName'); - const roleArn = config.getOptionalString('roleArn'); - const externalId = config.getOptionalString('externalId'); - const region = config.getString('region'); - - return new AWSIAMRoleProvider( - { accountId, roleName, roleArn, externalId, region }, - options, - ); + options: AWSIAMRoleProviderOptions, + ): AWSIAMRoleProvider { + const p = new AWSIAMRoleProvider(config, options); + + const defaultSchedule = { + frequency: { minutes: 120 }, + timeout: { minutes: 60 }, + initialDelay: { seconds: 30 }, + }; + + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : defaultSchedule; + + options.scheduler.scheduleTask({ + frequency: schedule.frequency, + timeout: schedule.timeout, + initialDelay: schedule.initialDelay, + id: 'aws-iam-role-entity-provider', + fn: p.run, + }); + + return p; } + /** [2] */ getProviderName(): string { return `aws-iam-role-${this.providerId ?? 0}`; } - private async getIam(dynamicAccountConfig?: DynamicAccountConfig) { - const { region } = this.getParsedConfig(dynamicAccountConfig); - const credentials = this.useTemporaryCredentials - ? this.getCredentials(dynamicAccountConfig) - : await this.getCredentialsProvider(); - return this.useTemporaryCredentials - ? new IAM({ credentials, region }) - : new IAM(credentials); + /** [3] */ + async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); + this.connection = connection; + this.scheduler.scheduleTask({ + frequency: { seconds: 5 }, + timeout: { seconds: 30 }, + id: 'aws-iam-role-entity-provider', + fn: this.run, + }); + await this.run(); + } + + private async getIam() { + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig( + this.config, + ); + const awsCredentialProvider = + await awsCredentialsManager.getCredentialProvider({ accountId }); + return new IAM({ + region, + credentialDefaultProvider: () => + awsCredentialProvider.sdkCredentialProvider, + }); } - async run(dynamicAccountConfig?: DynamicAccountConfig): Promise { + async run(): Promise { if (!this.connection) { + this.logger.info('Not initialized'); throw new Error('Not initialized'); } - const startTimestamp = process.hrtime(); - const { accountId } = this.getParsedConfig(dynamicAccountConfig); + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const startTimestamp = process.hrtime(); const groups = await this.getGroups(); this.logger.info(`Providing IAM role resources from AWS: ${accountId}`); const roleResources: ResourceEntity[] = []; - const defaultAnnotations = - this.buildDefaultAnnotations(dynamicAccountConfig); + const iam = await this.getIam(); - const iam = await this.getIam(dynamicAccountConfig); + const defaultAnnotations = this.buildDefaultAnnotations( + this.config, + accountId, + region, + ); const paginatorConfig = { client: iam, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.test.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.test.ts index 40af4c6ea..1387d1e11 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.test.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.test.ts @@ -14,21 +14,20 @@ * limitations under the License. */ +import { mockServices } from '@backstage/backend-test-utils'; import { IAM, ListUsersCommand, User } from '@aws-sdk/client-iam'; import { STS, GetCallerIdentityCommand } from '@aws-sdk/client-sts'; import { mockClient } from 'aws-sdk-client-mock'; -import { createLogger, transports } from 'winston'; import { AWSIAMUserProvider } from './AWSIAMUserProvider'; import { ConfigReader } from '@backstage/config'; -import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; const iam = mockClient(IAM); const sts = mockClient(STS); -const logger = createLogger({ - transports: [new transports.Console({ silent: true })], -}); +const logger = mockServices.logger.mock(); +const scheduler = mockServices.scheduler.mock(); describe('AWSIAMUserProvider', () => { const config = new ConfigReader({ @@ -53,7 +52,10 @@ describe('AWSIAMUserProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const provider = AWSIAMUserProvider.fromConfig(config, { logger }); + const provider = AWSIAMUserProvider.fromConfig(config, { + logger, + scheduler, + }); provider.connect(entityProviderConnection); await provider.run(); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ @@ -81,7 +83,10 @@ describe('AWSIAMUserProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const provider = AWSIAMUserProvider.fromConfig(config, { logger }); + const provider = AWSIAMUserProvider.fromConfig(config, { + logger, + scheduler, + }); provider.connect(entityProviderConnection); await provider.run(); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.ts index 93e9e506c..971fe78c3 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.ts @@ -14,9 +14,15 @@ * limitations under the License. */ +import { + LoggerService, + SchedulerService, + readSchedulerServiceTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-plugin-api'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { ANNOTATION_VIEW_URL, UserEntity } from '@backstage/catalog-model'; import { IAM, paginateListUsers } from '@aws-sdk/client-iam'; -import * as winston from 'winston'; +import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node'; import { Config } from '@backstage/config'; import { AWSEntityProvider } from './AWSEntityProvider'; import { ANNOTATION_AWS_IAM_USER_ARN } from '../annotations'; @@ -24,64 +30,106 @@ import { arnToName } from '../utils/arnToName'; import { ARN } from 'link2aws'; import { CatalogApi } from '@backstage/catalog-client'; import { LabelValueMapper } from '../utils/tags'; -import { DynamicAccountConfig } from '../types'; import { duration } from '../utils/timer'; +export type AWSIAMUserProviderOptions = { + logger: LoggerService; + scheduler: SchedulerService; + catalogApi?: CatalogApi; + providerId?: string; + ownerTag?: string; + useTemporaryCredentials?: boolean; + labelValueMapper?: LabelValueMapper; +}; + /** * Provides entities from AWS IAM User service. */ export class AWSIAMUserProvider extends AWSEntityProvider { + declare connection?: EntityProviderConnection; + + /** [1] */ static fromConfig( config: Config, - options: { - logger: winston.Logger; - catalogApi?: CatalogApi; - providerId?: string; - ownerTag?: string; - useTemporaryCredentials?: boolean; - labelValueMapper?: LabelValueMapper; - }, - ) { - const accountId = config.getString('accountId'); - const roleName = config.getString('roleName'); - const roleArn = config.getOptionalString('roleArn'); - const externalId = config.getOptionalString('externalId'); - const region = config.getString('region'); - - return new AWSIAMUserProvider( - { accountId, roleName, roleArn, externalId, region }, - options, - ); + options: AWSIAMUserProviderOptions, + ): AWSIAMUserProvider { + const p = new AWSIAMUserProvider(config, options); + + const defaultSchedule = { + frequency: { minutes: 120 }, + timeout: { minutes: 60 }, + initialDelay: { seconds: 30 }, + }; + + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : defaultSchedule; + + options.scheduler.scheduleTask({ + frequency: schedule.frequency, + timeout: schedule.timeout, + initialDelay: schedule.initialDelay, + id: 'aws-iam-user-entity-provider', + fn: p.run, + }); + + return p; } + /** [2] */ getProviderName(): string { return `aws-iam-user-${this.providerId ?? 0}`; } - private async getIam(dynamicAccountConfig?: DynamicAccountConfig) { - const { region } = this.getParsedConfig(dynamicAccountConfig); - const credentials = this.useTemporaryCredentials - ? this.getCredentials(dynamicAccountConfig) - : await this.getCredentialsProvider(); - return this.useTemporaryCredentials - ? new IAM({ credentials, region }) - : new IAM(credentials); + /** [3] */ + async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); + this.connection = connection; + this.scheduler.scheduleTask({ + frequency: { seconds: 5 }, + timeout: { seconds: 30 }, + id: 'aws-iam-user-entity-provider', + fn: this.run, + }); + await this.run(); + } + + private async getIam() { + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig( + this.config, + ); + const awsCredentialProvider = + await awsCredentialsManager.getCredentialProvider({ accountId }); + return new IAM({ + region, + credentialDefaultProvider: () => + awsCredentialProvider.sdkCredentialProvider, + }); } - async run(dynamicAccountConfig?: DynamicAccountConfig): Promise { + async run(): Promise { if (!this.connection) { throw new Error('Not initialized'); } + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const startTimestamp = process.hrtime(); - const { accountId } = this.getParsedConfig(dynamicAccountConfig); this.logger.info(`Providing IAM user resources from AWS: ${accountId}`); const userResources: UserEntity[] = []; - const defaultAnnotations = - this.buildDefaultAnnotations(dynamicAccountConfig); + const iam = await this.getIam(); - const iam = await this.getIam(dynamicAccountConfig); + const defaultAnnotations = this.buildDefaultAnnotations( + this.config, + accountId, + region, + ); const paginatorConfig = { client: iam, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.test.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.test.ts index 88023ff07..12ee9fd3a 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.test.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.test.ts @@ -14,21 +14,20 @@ * limitations under the License. */ +import { mockServices } from '@backstage/backend-test-utils'; import { Lambda, ListFunctionsCommand } from '@aws-sdk/client-lambda'; import { STS, GetCallerIdentityCommand } from '@aws-sdk/client-sts'; import { mockClient } from 'aws-sdk-client-mock'; -import { createLogger, transports } from 'winston'; import { ConfigReader } from '@backstage/config'; -import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { AWSLambdaFunctionProvider } from './AWSLambdaFunctionProvider'; const lambda = mockClient(Lambda); const sts = mockClient(STS); -const logger = createLogger({ - transports: [new transports.Console({ silent: true })], -}); +const logger = mockServices.logger.mock(); +const scheduler = mockServices.scheduler.mock(); describe('AWSLambdaFunctionProvider', () => { const config = new ConfigReader({ @@ -53,7 +52,10 @@ describe('AWSLambdaFunctionProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const provider = AWSLambdaFunctionProvider.fromConfig(config, { logger }); + const provider = AWSLambdaFunctionProvider.fromConfig(config, { + logger, + scheduler, + }); provider.connect(entityProviderConnection); await provider.run(); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ @@ -93,7 +95,10 @@ describe('AWSLambdaFunctionProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const provider = AWSLambdaFunctionProvider.fromConfig(config, { logger }); + const provider = AWSLambdaFunctionProvider.fromConfig(config, { + logger, + scheduler, + }); provider.connect(entityProviderConnection); await provider.run(); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ @@ -129,53 +134,5 @@ describe('AWSLambdaFunctionProvider', () => { ], }); }); - - it('is able to use dynamic config', async () => { - const entityProviderConnection: EntityProviderConnection = { - applyMutation: jest.fn(), - refresh: jest.fn(), - }; - const provider = AWSLambdaFunctionProvider.fromConfig(config, { - logger, - useTemporaryCredentials: true, - }); - await provider.connect(entityProviderConnection); - await provider.run({ - roleArn: 'arn:aws:iam::999999999999:role/dynamic-role', - region: 'us-east-1', - }); - expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - type: 'full', - entities: [ - expect.objectContaining({ - entity: expect.objectContaining({ - kind: 'Resource', - metadata: expect.objectContaining({ - annotations: { - 'amazon.com/iam-role-arn': - 'arn:aws:iam::123456789012:role/lambdaRole', - 'amazon.com/lambda-function-arn': - 'arn:aws:lambda:eu-west-1:123456789012:function:my-function', - 'backstage.io/managed-by-location': - 'aws-lambda-function-0:arn:aws:iam::999999999999:role/dynamic-role', - 'backstage.io/managed-by-origin-location': - 'aws-lambda-function-0:arn:aws:iam::999999999999:role/dynamic-role', - 'backstage.io/view-url': - 'https://eu-west-1.console.aws.amazon.com/lambda/home?region=eu-west-1#/functions/my-function', - }, - title: 'my-function', - architectures: ['x86_64'], - description: '', - ephemeralStorage: 512, - memorySize: 1024, - name: 'bc6fa48d05a0a464c5e2a5214985bd957578cd50314fc6076cef1845fadb3c8', - runtime: 'nodejs14.x', - timeout: 30, - }), - }), - }), - ], - }); - }); }); }); diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts index 1d3055c41..3e58e4af2 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts @@ -14,9 +14,15 @@ * limitations under the License. */ +import { + LoggerService, + SchedulerService, + readSchedulerServiceTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-plugin-api'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { ANNOTATION_VIEW_URL, ResourceEntity } from '@backstage/catalog-model'; import { Lambda, paginateListFunctions } from '@aws-sdk/client-lambda'; -import * as winston from 'winston'; +import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node'; import { Config } from '@backstage/config'; import { AWSEntityProvider } from './AWSEntityProvider'; import { @@ -31,69 +37,111 @@ import { relationshipsFromTags, } from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; -import { DynamicAccountConfig } from '../types'; import { duration } from '../utils/timer'; +export type AWSLambdaFunctionProviderOptions = { + logger: LoggerService; + scheduler: SchedulerService; + catalogApi?: CatalogApi; + providerId?: string; + ownerTag?: string; + useTemporaryCredentials?: boolean; + labelValueMapper?: LabelValueMapper; +}; + /** * Provides entities from AWS Lambda Function service. */ export class AWSLambdaFunctionProvider extends AWSEntityProvider { + declare connection?: EntityProviderConnection; + + /** [1] */ static fromConfig( config: Config, - options: { - logger: winston.Logger; - catalogApi?: CatalogApi; - providerId?: string; - ownerTag?: string; - useTemporaryCredentials?: boolean; - labelValueMapper?: LabelValueMapper; - }, - ) { - const accountId = config.getString('accountId'); - const roleName = config.getString('roleName'); - const roleArn = config.getOptionalString('roleArn'); - const externalId = config.getOptionalString('externalId'); - const region = config.getString('region'); - - return new AWSLambdaFunctionProvider( - { accountId, roleName, roleArn, externalId, region }, - options, - ); + options: AWSLambdaFunctionProviderOptions, + ): AWSLambdaFunctionProvider { + const p = new AWSLambdaFunctionProvider(config, options); + + const defaultSchedule = { + frequency: { minutes: 120 }, + timeout: { minutes: 60 }, + initialDelay: { seconds: 30 }, + }; + + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : defaultSchedule; + + options.scheduler.scheduleTask({ + frequency: schedule.frequency, + timeout: schedule.timeout, + initialDelay: schedule.initialDelay, + id: 'aws-lambda-function-entity-provider', + fn: p.run, + }); + + return p; } + /** [2] */ getProviderName(): string { return `aws-lambda-function-${this.providerId ?? 0}`; } - private async getLambda(dynamicAccountConfig?: DynamicAccountConfig) { - const { region } = this.getParsedConfig(dynamicAccountConfig); - const credentials = this.useTemporaryCredentials - ? this.getCredentials(dynamicAccountConfig) - : await this.getCredentialsProvider(); - return this.useTemporaryCredentials - ? new Lambda({ credentials, region }) - : new Lambda(credentials); + /** [3] */ + async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); + this.connection = connection; + this.scheduler.scheduleTask({ + frequency: { seconds: 5 }, + timeout: { seconds: 30 }, + id: 'aws-lambda-function-entity-provider', + fn: this.run, + }); + await this.run(); } - async run(dynamicAccountConfig?: DynamicAccountConfig): Promise { + private async getLambda() { + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig( + this.config, + ); + const awsCredentialProvider = + await awsCredentialsManager.getCredentialProvider({ accountId }); + return new Lambda({ + region, + credentialDefaultProvider: () => + awsCredentialProvider.sdkCredentialProvider, + }); + } + + /** [4] */ + async run(): Promise { if (!this.connection) { + this.logger.info('Not initialized'); throw new Error('Not initialized'); } - const startTimestamp = process.hrtime(); - const { accountId } = this.getParsedConfig(dynamicAccountConfig); + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const startTimestamp = process.hrtime(); const groups = await this.getGroups(); this.logger.info( `Providing lambda function resources from AWS: ${accountId}`, ); - const lambdaComponents: ResourceEntity[] = []; - const lambda = await this.getLambda(dynamicAccountConfig); + const lambda = await this.getLambda(); - const defaultAnnotations = - this.buildDefaultAnnotations(dynamicAccountConfig); + const defaultAnnotations = this.buildDefaultAnnotations( + this.config, + accountId, + region, + ); const paginatorConfig = { client: lambda, @@ -113,7 +161,7 @@ export class AWSLambdaFunctionProvider extends AWSEntityProvider { }); tags = tagsResponse?.Tags ?? {}; } catch (e) { - this.logger.warn('Unable to get tags for Lambda functions', e); + this.logger.warn(`Unable to get tags for Lambda functions\n{e}`); } const annotations: { [name: string]: string } = { diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts index 0a38aef4b..e4371e877 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts @@ -14,14 +14,19 @@ * limitations under the License. */ +import { + LoggerService, + SchedulerService, + readSchedulerServiceTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-plugin-api'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { ResourceEntity } from '@backstage/catalog-model'; - import { OrganizationsClient, paginateListAccounts, paginateListTagsForResource, } from '@aws-sdk/client-organizations'; -import * as winston from 'winston'; +import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node'; import { Config } from '@backstage/config'; import { AWSEntityProvider } from './AWSEntityProvider'; import { @@ -36,62 +41,94 @@ import { } from '../utils/tags'; import { Tag } from '@aws-sdk/client-organizations/dist-types/models/models_0'; import { CatalogApi } from '@backstage/catalog-client'; -import { DynamicAccountConfig } from '../types'; import { duration } from '../utils/timer'; +export type AWSOrganizationAccountsProviderOptions = { + logger: LoggerService; + scheduler: SchedulerService; + catalogApi?: CatalogApi; + providerId?: string; + ownerTag?: string; + useTemporaryCredentials?: boolean; + labelValueMapper?: LabelValueMapper; +}; + /** * Provides entities from AWS Organizations accounts. */ export class AWSOrganizationAccountsProvider extends AWSEntityProvider { + /** [1] */ static fromConfig( config: Config, - options: { - logger: winston.Logger; - catalogApi?: CatalogApi; - providerId?: string; - ownerTag?: string; - useTemporaryCredentials?: boolean; - labelValueMapper?: LabelValueMapper; - }, - ) { - const accountId = config.getString('accountId'); - const roleName = config.getString('roleName'); - const roleArn = config.getOptionalString('roleArn'); - const externalId = config.getOptionalString('externalId'); - const region = config.getString('region'); - - return new AWSOrganizationAccountsProvider( - { accountId, roleName, roleArn, externalId, region }, - options, - ); + options: AWSOrganizationAccountsProviderOptions, + ): AWSOrganizationAccountsProvider { + const p = new AWSOrganizationAccountsProvider(config, options); + + const defaultSchedule = { + frequency: { minutes: 120 }, + timeout: { minutes: 60 }, + initialDelay: { seconds: 30 }, + }; + + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : defaultSchedule; + + options.scheduler.scheduleTask({ + frequency: schedule.frequency, + timeout: schedule.timeout, + initialDelay: schedule.initialDelay, + id: 'aws-organization-accounts-entity-provider', + fn: p.run, + }); + + return p; } + /** [2] */ getProviderName(): string { return `aws-organization-accounts-${this.providerId ?? 0}`; } - private async getOrganizationsClient( - dynamicAccountConfig?: DynamicAccountConfig, - ) { - const { region } = this.getParsedConfig(dynamicAccountConfig); - const credentials = this.useTemporaryCredentials - ? this.getCredentials(dynamicAccountConfig) - : await this.getCredentialsProvider(); - return this.useTemporaryCredentials - ? new OrganizationsClient({ - credentials, - region, - }) - : new OrganizationsClient(credentials); + /** [3] */ + async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); + this.connection = connection; + this.scheduler.scheduleTask({ + frequency: { seconds: 5 }, + timeout: { seconds: 30 }, + id: 'aws-organization-accounts-entity-provider', + fn: this.run, + }); + await this.run(); } - async run(dynamicAccountConfig?: DynamicAccountConfig): Promise { + private async getOrganizationsClient() { + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig( + this.config, + ); + const awsCredentialProvider = + await awsCredentialsManager.getCredentialProvider({ accountId }); + return new OrganizationsClient({ + region, + credentialDefaultProvider: () => + awsCredentialProvider.sdkCredentialProvider, + }); + } + + /** [4] */ + async run(): Promise { if (!this.connection) { throw new Error('Not initialized'); } - const startTimestamp = process.hrtime(); - const { accountId } = this.getParsedConfig(dynamicAccountConfig); + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const startTimestamp = process.hrtime(); const groups = await this.getGroups(); this.logger.info( @@ -99,12 +136,13 @@ export class AWSOrganizationAccountsProvider extends AWSEntityProvider { ); const accountResources: ResourceEntity[] = []; - const organizationsClient = await this.getOrganizationsClient( - dynamicAccountConfig, - ); + const organizationsClient = await this.getOrganizationsClient(); - const defaultAnnotations = - this.buildDefaultAnnotations(dynamicAccountConfig); + const defaultAnnotations = this.buildDefaultAnnotations( + this.config, + accountId, + region, + ); const paginatorConfig = { client: organizationsClient, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts index f013ab00e..93731f8e4 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts @@ -14,9 +14,15 @@ * limitations under the License. */ +import { + LoggerService, + SchedulerService, + readSchedulerServiceTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-plugin-api'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { ANNOTATION_VIEW_URL, ResourceEntity } from '@backstage/catalog-model'; import { RDS, paginateDescribeDBInstances } from '@aws-sdk/client-rds'; -import * as winston from 'winston'; +import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node'; import { Config } from '@backstage/config'; import { AWSEntityProvider } from './AWSEntityProvider'; import { ANNOTATION_AWS_RDS_INSTANCE_ARN } from '../annotations'; @@ -27,64 +33,106 @@ import { relationshipsFromTags, } from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; -import { DynamicAccountConfig } from '../types'; import { duration } from '../utils/timer'; +export type AWSRDSProviderOptions = { + logger: LoggerService; + scheduler: SchedulerService; + catalogApi?: CatalogApi; + providerId?: string; + ownerTag?: string; + useTemporaryCredentials?: boolean; + labelValueMapper?: LabelValueMapper; +}; + /** * Provides entities from AWS Relational Database Service. */ export class AWSRDSProvider extends AWSEntityProvider { + /** [1] */ static fromConfig( config: Config, - options: { - logger: winston.Logger; - catalogApi?: CatalogApi; - providerId?: string; - useTemporaryCredentials?: boolean; - labelValueMapper?: LabelValueMapper; - }, - ) { - const accountId = config.getString('accountId'); - const roleName = config.getString('roleName'); - const roleArn = config.getOptionalString('roleArn'); - const externalId = config.getOptionalString('externalId'); - const region = config.getString('region'); - - return new AWSRDSProvider( - { accountId, roleName, roleArn, externalId, region }, - options, - ); + options: AWSRDSProviderOptions, + ): AWSRDSProvider { + const p = new AWSRDSProvider(config, options); + + const defaultSchedule = { + frequency: { minutes: 120 }, + timeout: { minutes: 60 }, + initialDelay: { seconds: 30 }, + }; + + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : defaultSchedule; + + options.scheduler.scheduleTask({ + frequency: schedule.frequency, + timeout: schedule.timeout, + initialDelay: schedule.initialDelay, + id: 'amazon-rds-entity-provider', + fn: p.run, + }); + + return p; } + //* * [2] */ getProviderName(): string { - return `aws-rds-provider-${this.providerId ?? 0}`; + return `aws-rds-${this.providerId ?? 0}`; } - private async getRdsClient(dynamicAccountConfig?: DynamicAccountConfig) { - const { region } = this.getParsedConfig(dynamicAccountConfig); - const credentials = this.useTemporaryCredentials - ? this.getCredentials(dynamicAccountConfig) - : await this.getCredentialsProvider(); - return this.useTemporaryCredentials - ? new RDS({ credentials, region }) - : new RDS(credentials); + /** [3] */ + async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); + this.connection = connection; + this.scheduler.scheduleTask({ + frequency: { seconds: 5 }, + timeout: { seconds: 30 }, + id: 'amazon-rds-entity-provider', + fn: this.run, + }); + await this.run(); + } + + private async getRdsClient() { + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig( + this.config, + ); + const awsCredentialProvider = + await awsCredentialsManager.getCredentialProvider({ accountId }); + return new RDS({ + region, + credentialDefaultProvider: () => + awsCredentialProvider.sdkCredentialProvider, + }); } - async run(dynamicAccountConfig?: DynamicAccountConfig): Promise { + /** [4] */ + async run(): Promise { if (!this.connection) { throw new Error('Not initialized'); } - const startTimestamp = process.hrtime(); - const { accountId } = this.getParsedConfig(dynamicAccountConfig); + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const startTimestamp = process.hrtime(); const groups = await this.getGroups(); + this.logger.info(`Providing RDS resources from AWS: ${accountId}`); const rdsResources: ResourceEntity[] = []; - const rdsClient = await this.getRdsClient(dynamicAccountConfig); + const rdsClient = await this.getRdsClient(); - const defaultAnnotations = - this.buildDefaultAnnotations(dynamicAccountConfig); + const defaultAnnotations = this.buildDefaultAnnotations( + this.config, + accountId, + region, + ); const paginatorConfig = { client: rdsClient, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.test.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.test.ts index 8411678c5..a4fb7cdf1 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.test.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.test.ts @@ -14,21 +14,20 @@ * limitations under the License. */ +import { mockServices } from '@backstage/backend-test-utils'; import { S3, ListBucketsCommand } from '@aws-sdk/client-s3'; import { STS, GetCallerIdentityCommand } from '@aws-sdk/client-sts'; import { mockClient } from 'aws-sdk-client-mock'; -import { createLogger, transports } from 'winston'; import { ConfigReader } from '@backstage/config'; -import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { AWSS3BucketProvider } from './AWSS3BucketProvider'; const s3 = mockClient(S3); const sts = mockClient(STS); -const logger = createLogger({ - transports: [new transports.Console({ silent: true })], -}); +const logger = mockServices.logger.mock(); +const scheduler = mockServices.scheduler.mock(); describe('AWSS3BucketProvider', () => { const config = new ConfigReader({ @@ -53,7 +52,10 @@ describe('AWSS3BucketProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const provider = AWSS3BucketProvider.fromConfig(config, { logger }); + const provider = AWSS3BucketProvider.fromConfig(config, { + logger, + scheduler, + }); provider.connect(entityProviderConnection); await provider.run(); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ @@ -80,7 +82,10 @@ describe('AWSS3BucketProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const provider = AWSS3BucketProvider.fromConfig(config, { logger }); + const provider = AWSS3BucketProvider.fromConfig(config, { + logger, + scheduler, + }); provider.connect(entityProviderConnection); await provider.run(); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts index 14439a3c7..866243b71 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts @@ -14,9 +14,15 @@ * limitations under the License. */ +import { + LoggerService, + SchedulerService, + readSchedulerServiceTaskScheduleDefinitionFromConfig, +} from '@backstage/backend-plugin-api'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { ANNOTATION_VIEW_URL, ResourceEntity } from '@backstage/catalog-model'; import { S3, Tag } from '@aws-sdk/client-s3'; -import * as winston from 'winston'; +import { DefaultAwsCredentialsManager } from '@backstage/integration-aws-node'; import { Config } from '@backstage/config'; import { AWSEntityProvider } from './AWSEntityProvider'; import { ANNOTATION_AWS_S3_BUCKET_ARN } from '../annotations'; @@ -28,65 +34,107 @@ import { relationshipsFromTags, } from '../utils/tags'; import { CatalogApi } from '@backstage/catalog-client'; -import { DynamicAccountConfig } from '../types'; import { duration } from '../utils/timer'; +export type AWSS3BucketProviderOptions = { + logger: LoggerService; + scheduler: SchedulerService; + catalogApi?: CatalogApi; + providerId?: string; + ownerTag?: string; + useTemporaryCredentials?: boolean; + labelValueMapper?: LabelValueMapper; +}; + /** * Provides entities from AWS S3 Bucket service. */ export class AWSS3BucketProvider extends AWSEntityProvider { + /** [1] */ static fromConfig( config: Config, - options: { - logger: winston.Logger; - catalogApi?: CatalogApi; - providerId?: string; - ownerTag?: string; - useTemporaryCredentials?: boolean; - labelValueMapper?: LabelValueMapper; - }, - ) { - const accountId = config.getString('accountId'); - const roleName = config.getString('roleName'); - const roleArn = config.getOptionalString('roleArn'); - const externalId = config.getOptionalString('externalId'); - const region = config.getString('region'); - - return new AWSS3BucketProvider( - { accountId, roleName, roleArn, externalId, region }, - options, - ); + options: AWSS3BucketProviderOptions, + ): AWSS3BucketProvider { + const p = new AWSS3BucketProvider(config, options); + + const defaultSchedule = { + frequency: { minutes: 120 }, + timeout: { minutes: 60 }, + initialDelay: { seconds: 30 }, + }; + + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : defaultSchedule; + + options.scheduler.scheduleTask({ + frequency: schedule.frequency, + timeout: schedule.timeout, + initialDelay: schedule.initialDelay, + id: 'amazon-s3-bucket-entity-provider', + fn: p.run, + }); + + return p; } + /** [2] */ getProviderName(): string { - return `aws-s3-bucket-${this.providerId ?? 0}`; + return `amazon-s3-bucket-${this.providerId ?? 0}`; } - private async getS3(dynamicAccountConfig?: DynamicAccountConfig) { - const { region } = this.getParsedConfig(dynamicAccountConfig); - const credentials = this.useTemporaryCredentials - ? this.getCredentials(dynamicAccountConfig) - : await this.getCredentialsProvider(); - return this.useTemporaryCredentials - ? new S3({ credentials, region }) - : new S3(credentials); + /** [3] */ + async connect(connection: EntityProviderConnection): Promise { + this.logger.info('connecting'); + this.connection = connection; + this.scheduler.scheduleTask({ + frequency: { seconds: 5 }, + timeout: { seconds: 30 }, + id: 'amazon-s3-bucket-entity-provider', + fn: this.run, + }); + await this.run(); } - async run(dynamicAccountConfig?: DynamicAccountConfig): Promise { + private async getS3() { + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const awsCredentialsManager = DefaultAwsCredentialsManager.fromConfig( + this.config, + ); + const awsCredentialProvider = + await awsCredentialsManager.getCredentialProvider({ accountId }); + return new S3({ + region, + credentialDefaultProvider: () => + awsCredentialProvider.sdkCredentialProvider, + }); + } + + /** [4] */ + async run(): Promise { if (!this.connection) { + this.logger.info('Not initialized'); throw new Error('Not initialized'); } + const accountId = this.config.getString('accountId'); + const region = this.config.getOptionalString('region') || 'us-east-1'; + const startTimestamp = process.hrtime(); - const { accountId } = this.getParsedConfig(dynamicAccountConfig); const groups = await this.getGroups(); this.logger.info(`Providing S3 bucket resources from AWS: ${accountId}`); const s3Resources: ResourceEntity[] = []; - const s3 = await this.getS3(dynamicAccountConfig); + const s3 = await this.getS3(); - const defaultAnnotations = - this.buildDefaultAnnotations(dynamicAccountConfig); + const defaultAnnotations = this.buildDefaultAnnotations( + this.config, + accountId, + region, + ); const buckets = await s3.listBuckets({}); diff --git a/plugins/backend/catalog-backend-module-aws/src/types.ts b/plugins/backend/catalog-backend-module-aws/src/types.ts index 871c8221b..1e11a2948 100644 --- a/plugins/backend/catalog-backend-module-aws/src/types.ts +++ b/plugins/backend/catalog-backend-module-aws/src/types.ts @@ -81,10 +81,14 @@ export interface AWSAccountProviderConfig { export type AccountConfig = { accountId: string; - roleName: string; + accessKeyId?: string; + secretAccessKey?: string; + profile?: string; + roleName?: string; roleArn?: string; + partition?: string; + region?: string; externalId?: string; - region: string; }; export type DynamicAccountConfig = { diff --git a/yarn.lock b/yarn.lock index caed79319..3e20e1bd1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14590,9 +14590,9 @@ integrity sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ== "@types/react-dom@*", "@types/react-dom@<18.0.0", "@types/react-dom@^18", "@types/react-dom@^18.0.0": - version "18.3.0" - resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-18.3.0.tgz#0cbc818755d87066ab6ca74fbedb2547d74a82b0" - integrity sha512-EhwApuTmMBmXuFOikhQLIBUn6uFg81SwLMOAUgodJF14SOBOCMdU04gDoYi0WOJJHD144TL32z4yDqCW3dnkQg== + version "18.3.1" + resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-18.3.1.tgz#1e4654c08a9cdcfb6594c780ac59b55aad42fe07" + integrity sha512-qW1Mfv8taImTthu4KoXgDfLuk4bydU6Q/TkADnDWWHwi4NX4BR+LWfTp2sVmTqRrsHvyDDTelgelxJ+SsejKKQ== dependencies: "@types/react" "*" @@ -14628,9 +14628,9 @@ "@types/react" "*" "@types/react@*", "@types/react@^16.13.1 || ^17.0.0", "@types/react@^16.13.1 || ^17.0.0 || ^18.0.0", "@types/react@^18": - version "18.3.11" - resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.11.tgz#9d530601ff843ee0d7030d4227ea4360236bd537" - integrity sha512-r6QZ069rFTjrEYgFdOck1gK7FLVsgJE7tTz0pQBczlBNUhBNk0MQH4UbnFSwjpQLMkLzgqvBBa+qGpLje16eTQ== + version "18.3.12" + resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.12.tgz#99419f182ccd69151813b7ee24b792fe08774f60" + integrity sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw== dependencies: "@types/prop-types" "*" csstype "^3.0.2" @@ -31871,16 +31871,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -31954,7 +31945,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -31968,13 +31959,6 @@ strip-ansi@5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -34174,7 +34158,7 @@ wordwrap@^1.0.0: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -34192,15 +34176,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.0.1, wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 33c83924d13fd8f9a34ca4ea4be2ac9500cd7850 Mon Sep 17 00:00:00 2001 From: Florian JUDITH Date: Fri, 1 Nov 2024 01:36:55 -0400 Subject: [PATCH 2/6] Updated yarn.lock --- yarn.lock | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index 3e20e1bd1..25dac5676 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14589,13 +14589,20 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.7.tgz#50ae4353eaaddc04044279812f52c8c65857dbcb" integrity sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ== -"@types/react-dom@*", "@types/react-dom@<18.0.0", "@types/react-dom@^18", "@types/react-dom@^18.0.0": +"@types/react-dom@*", "@types/react-dom@^18.0.0": version "18.3.1" resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-18.3.1.tgz#1e4654c08a9cdcfb6594c780ac59b55aad42fe07" integrity sha512-qW1Mfv8taImTthu4KoXgDfLuk4bydU6Q/TkADnDWWHwi4NX4BR+LWfTp2sVmTqRrsHvyDDTelgelxJ+SsejKKQ== dependencies: "@types/react" "*" +"@types/react-dom@<18.0.0": + version "17.0.25" + resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-17.0.25.tgz#e0e5b3571e1069625b3a3da2b279379aa33a0cb5" + integrity sha512-urx7A7UxkZQmThYA4So0NelOVjx3V4rNFVJwp0WZlbIK5eM4rNJDiN3R/E9ix0MBh6kAEojk/9YL+Te6D9zHNA== + dependencies: + "@types/react" "^17" + "@types/react-redux@^7.1.20": version "7.1.33" resolved "https://registry.yarnpkg.com/@types/react-redux/-/react-redux-7.1.33.tgz#53c5564f03f1ded90904e3c90f77e4bd4dc20b15" @@ -14627,7 +14634,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@^16.13.1 || ^17.0.0", "@types/react@^16.13.1 || ^17.0.0 || ^18.0.0", "@types/react@^18": +"@types/react@*", "@types/react@^16.13.1 || ^17.0.0 || ^18.0.0": version "18.3.12" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.12.tgz#99419f182ccd69151813b7ee24b792fe08774f60" integrity sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw== @@ -14635,6 +14642,15 @@ "@types/prop-types" "*" csstype "^3.0.2" +"@types/react@^16.13.1 || ^17.0.0", "@types/react@^17": + version "17.0.83" + resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.83.tgz#b477c56387b74279281149dcf5ba2a1e2216d131" + integrity sha512-l0m4ArKJvmFtR4e8UmKrj1pB4tUgOhJITf+mADyF/p69Ts1YAR/E+G9XEM0mHXKVRa1dQNHseyyDNzeuAXfXQw== + dependencies: + "@types/prop-types" "*" + "@types/scheduler" "^0.16" + csstype "^3.0.2" + "@types/recharts@^1.8.14": version "1.8.29" resolved "https://registry.yarnpkg.com/@types/recharts/-/recharts-1.8.29.tgz#5e117521a65bf015b808350b45b65553ff5011f3" @@ -14682,6 +14698,11 @@ dependencies: htmlparser2 "^4.1.0" +"@types/scheduler@^0.16": + version "0.16.8" + resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.8.tgz#ce5ace04cfeabe7ef87c0091e50752e36707deff" + integrity sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A== + "@types/semver@7.5.8", "@types/semver@^7.5.0": version "7.5.8" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e" From 69356d6c43e3f107d9c3ff8f88b5b4e94bfa8836 Mon Sep 17 00:00:00 2001 From: Florian JUDITH Date: Fri, 1 Nov 2024 01:37:05 -0400 Subject: [PATCH 3/6] Updated yarn.lock --- yarn.lock | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/yarn.lock b/yarn.lock index 25dac5676..3e20e1bd1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14589,20 +14589,13 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.7.tgz#50ae4353eaaddc04044279812f52c8c65857dbcb" integrity sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ== -"@types/react-dom@*", "@types/react-dom@^18.0.0": +"@types/react-dom@*", "@types/react-dom@<18.0.0", "@types/react-dom@^18", "@types/react-dom@^18.0.0": version "18.3.1" resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-18.3.1.tgz#1e4654c08a9cdcfb6594c780ac59b55aad42fe07" integrity sha512-qW1Mfv8taImTthu4KoXgDfLuk4bydU6Q/TkADnDWWHwi4NX4BR+LWfTp2sVmTqRrsHvyDDTelgelxJ+SsejKKQ== dependencies: "@types/react" "*" -"@types/react-dom@<18.0.0": - version "17.0.25" - resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-17.0.25.tgz#e0e5b3571e1069625b3a3da2b279379aa33a0cb5" - integrity sha512-urx7A7UxkZQmThYA4So0NelOVjx3V4rNFVJwp0WZlbIK5eM4rNJDiN3R/E9ix0MBh6kAEojk/9YL+Te6D9zHNA== - dependencies: - "@types/react" "^17" - "@types/react-redux@^7.1.20": version "7.1.33" resolved "https://registry.yarnpkg.com/@types/react-redux/-/react-redux-7.1.33.tgz#53c5564f03f1ded90904e3c90f77e4bd4dc20b15" @@ -14634,7 +14627,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@^16.13.1 || ^17.0.0 || ^18.0.0": +"@types/react@*", "@types/react@^16.13.1 || ^17.0.0", "@types/react@^16.13.1 || ^17.0.0 || ^18.0.0", "@types/react@^18": version "18.3.12" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.12.tgz#99419f182ccd69151813b7ee24b792fe08774f60" integrity sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw== @@ -14642,15 +14635,6 @@ "@types/prop-types" "*" csstype "^3.0.2" -"@types/react@^16.13.1 || ^17.0.0", "@types/react@^17": - version "17.0.83" - resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.83.tgz#b477c56387b74279281149dcf5ba2a1e2216d131" - integrity sha512-l0m4ArKJvmFtR4e8UmKrj1pB4tUgOhJITf+mADyF/p69Ts1YAR/E+G9XEM0mHXKVRa1dQNHseyyDNzeuAXfXQw== - dependencies: - "@types/prop-types" "*" - "@types/scheduler" "^0.16" - csstype "^3.0.2" - "@types/recharts@^1.8.14": version "1.8.29" resolved "https://registry.yarnpkg.com/@types/recharts/-/recharts-1.8.29.tgz#5e117521a65bf015b808350b45b65553ff5011f3" @@ -14698,11 +14682,6 @@ dependencies: htmlparser2 "^4.1.0" -"@types/scheduler@^0.16": - version "0.16.8" - resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.8.tgz#ce5ace04cfeabe7ef87c0091e50752e36707deff" - integrity sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A== - "@types/semver@7.5.8", "@types/semver@^7.5.0": version "7.5.8" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e" From aaa65a36f13b2db721ba2db4bb1f089259c115b4 Mon Sep 17 00:00:00 2001 From: Florian JUDITH Date: Fri, 1 Nov 2024 01:39:58 -0400 Subject: [PATCH 4/6] Added changeset --- .changeset/tricky-hairs-double.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/tricky-hairs-double.md diff --git a/.changeset/tricky-hairs-double.md b/.changeset/tricky-hairs-double.md new file mode 100644 index 000000000..2fb744664 --- /dev/null +++ b/.changeset/tricky-hairs-double.md @@ -0,0 +1,5 @@ +--- +'@roadiehq/catalog-backend-module-aws': minor +--- + +Added support for new backend system and full support of `@backtage/integration-aws-node` for session management From 876743691905a4dc4f7ed964fa405d37bd7127c3 Mon Sep 17 00:00:00 2001 From: Florian JUDITH Date: Fri, 1 Nov 2024 08:05:34 -0400 Subject: [PATCH 5/6] Fixed typos --- plugins/backend/catalog-backend-module-aws/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/backend/catalog-backend-module-aws/README.md b/plugins/backend/catalog-backend-module-aws/README.md index ca25ad844..f54c81ef0 100644 --- a/plugins/backend/catalog-backend-module-aws/README.md +++ b/plugins/backend/catalog-backend-module-aws/README.md @@ -37,7 +37,7 @@ import { AWSS3BucketProvider, AWSIAMUserProvider, AWSEC2Provider, -} from '@internal/catalog-backend-module-aws'; +} from '@roadihq/catalog-backend-module-aws'; export default async function createPlugin( env: PluginEnvironment, From ba719b7cc0d8fe60cdc82a79404e6fc760e0036f Mon Sep 17 00:00:00 2001 From: Florian JUDITH Date: Fri, 1 Nov 2024 11:37:55 -0400 Subject: [PATCH 6/6] Fixed default schedule frequency --- .../src/providers/AWSDynamoDbTableDataProvider.ts | 2 +- .../src/providers/AWSDynamoDbTableProvider.ts | 4 +++- .../src/providers/AWSEC2Provider.ts | 4 +++- .../src/providers/AWSEKSClusterProvider.ts | 3 ++- .../src/providers/AWSIAMRoleProvider.ts | 2 +- .../src/providers/AWSIAMUserProvider.ts | 2 +- .../src/providers/AWSLambdaFunctionProvider.ts | 2 +- .../src/providers/AWSOrganizationAccountsProvider.ts | 4 +++- .../src/providers/AWSRDSProvider.ts | 4 +++- .../src/providers/AWSS3BucketProvider.ts | 4 +++- 10 files changed, 21 insertions(+), 10 deletions(-) diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.ts index 79c066327..1a6b76e2c 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableDataProvider.ts @@ -118,7 +118,7 @@ export class AWSDynamoDbTableDataProvider implements EntityProvider { this.logger.info('connecting'); this.connection = connection; this.scheduler.scheduleTask({ - frequency: { seconds: 5 }, + frequency: { minutes: 5 }, timeout: { seconds: 30 }, id: 'sync-mta-catalog', fn: this.run, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts index 24ba85b7a..beb18f96c 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSDynamoDbTableProvider.ts @@ -49,6 +49,8 @@ export type AWSDynamoDbTableProviderOptions = { * Provides entities from AWS DynamoDB service. */ export class AWSDynamoDbTableProvider extends AWSEntityProvider { + declare connection?: EntityProviderConnection; + /** [1] */ static fromConfig( config: Config, @@ -89,7 +91,7 @@ export class AWSDynamoDbTableProvider extends AWSEntityProvider { this.logger.info('connecting'); this.connection = connection; this.scheduler.scheduleTask({ - frequency: { seconds: 5 }, + frequency: { minutes: 5 }, timeout: { seconds: 30 }, id: 'amazon-s3-bucket-entity-provider', fn: this.run, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts index d8d2884b0..44c77c31d 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEC2Provider.ts @@ -49,6 +49,8 @@ export type AWSEC2ProviderOptions = { * Provides entities from AWS Elastic Compute Cloud. */ export class AWSEC2Provider extends AWSEntityProvider { + declare connection?: EntityProviderConnection; + /** [1] */ static fromConfig( config: Config, @@ -89,7 +91,7 @@ export class AWSEC2Provider extends AWSEntityProvider { this.logger.info('connecting'); this.connection = connection; this.scheduler.scheduleTask({ - frequency: { seconds: 5 }, + frequency: { minutes: 5 }, timeout: { seconds: 30 }, id: 'amazon-ec2-entity-provider', fn: this.run, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts index bf19bff6b..191df2536 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSEKSClusterProvider.ts @@ -60,6 +60,7 @@ export type AWSEKSClusterProviderOptions = { * Provides entities from AWS EKS Cluster service. */ export class AWSEKSClusterProvider extends AWSEntityProvider { + declare connection?: EntityProviderConnection; private readonly clusterTypeValue: string; //* * [1] */ @@ -119,7 +120,7 @@ export class AWSEKSClusterProvider extends AWSEntityProvider { this.logger.info('connecting'); this.connection = connection; this.scheduler.scheduleTask({ - frequency: { seconds: 5 }, + frequency: { minutes: 5 }, timeout: { seconds: 30 }, id: 'amazon-eks-cluster-entity-provider', fn: this.run, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts index 56d47428f..bb1f0dcb1 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMRoleProvider.ts @@ -92,7 +92,7 @@ export class AWSIAMRoleProvider extends AWSEntityProvider { this.logger.info('connecting'); this.connection = connection; this.scheduler.scheduleTask({ - frequency: { seconds: 5 }, + frequency: { minutes: 5 }, timeout: { seconds: 30 }, id: 'aws-iam-role-entity-provider', fn: this.run, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.ts index 971fe78c3..1170b09d0 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSIAMUserProvider.ts @@ -88,7 +88,7 @@ export class AWSIAMUserProvider extends AWSEntityProvider { this.logger.info('connecting'); this.connection = connection; this.scheduler.scheduleTask({ - frequency: { seconds: 5 }, + frequency: { minutes: 5 }, timeout: { seconds: 30 }, id: 'aws-iam-user-entity-provider', fn: this.run, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts index 3e58e4af2..09daf2b13 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSLambdaFunctionProvider.ts @@ -95,7 +95,7 @@ export class AWSLambdaFunctionProvider extends AWSEntityProvider { this.logger.info('connecting'); this.connection = connection; this.scheduler.scheduleTask({ - frequency: { seconds: 5 }, + frequency: { minutes: 5 }, timeout: { seconds: 30 }, id: 'aws-lambda-function-entity-provider', fn: this.run, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts index e4371e877..e3aab9b0e 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSOrganizationAccountsProvider.ts @@ -57,6 +57,8 @@ export type AWSOrganizationAccountsProviderOptions = { * Provides entities from AWS Organizations accounts. */ export class AWSOrganizationAccountsProvider extends AWSEntityProvider { + declare connection?: EntityProviderConnection; + /** [1] */ static fromConfig( config: Config, @@ -97,7 +99,7 @@ export class AWSOrganizationAccountsProvider extends AWSEntityProvider { this.logger.info('connecting'); this.connection = connection; this.scheduler.scheduleTask({ - frequency: { seconds: 5 }, + frequency: { minutes: 5 }, timeout: { seconds: 30 }, id: 'aws-organization-accounts-entity-provider', fn: this.run, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts index 93731f8e4..78e1ab6b5 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSRDSProvider.ts @@ -49,6 +49,8 @@ export type AWSRDSProviderOptions = { * Provides entities from AWS Relational Database Service. */ export class AWSRDSProvider extends AWSEntityProvider { + declare connection?: EntityProviderConnection; + /** [1] */ static fromConfig( config: Config, @@ -89,7 +91,7 @@ export class AWSRDSProvider extends AWSEntityProvider { this.logger.info('connecting'); this.connection = connection; this.scheduler.scheduleTask({ - frequency: { seconds: 5 }, + frequency: { minutes: 5 }, timeout: { seconds: 30 }, id: 'amazon-rds-entity-provider', fn: this.run, diff --git a/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts b/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts index 866243b71..aec4cb6c4 100644 --- a/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts +++ b/plugins/backend/catalog-backend-module-aws/src/providers/AWSS3BucketProvider.ts @@ -50,6 +50,8 @@ export type AWSS3BucketProviderOptions = { * Provides entities from AWS S3 Bucket service. */ export class AWSS3BucketProvider extends AWSEntityProvider { + declare connection?: EntityProviderConnection; + /** [1] */ static fromConfig( config: Config, @@ -90,7 +92,7 @@ export class AWSS3BucketProvider extends AWSEntityProvider { this.logger.info('connecting'); this.connection = connection; this.scheduler.scheduleTask({ - frequency: { seconds: 5 }, + frequency: { minutes: 5 }, timeout: { seconds: 30 }, id: 'amazon-s3-bucket-entity-provider', fn: this.run,