diff --git a/cid/builtin/core/data/queries/shared/account_map.sql b/cid/builtin/core/data/queries/shared/account_map.sql index eb5c4dfc..a6b96f2b 100644 --- a/cid/builtin/core/data/queries/shared/account_map.sql +++ b/cid/builtin/core/data/queries/shared/account_map.sql @@ -1,10 +1,12 @@ CREATE OR REPLACE VIEW account_map AS SELECT ${account_id} AS account_id, - MAX_BY(${account_name}, concat(${account_name}, ': ', ${account_id}) AS account_name -FROM + MAX_BY(${account_name}, concat(${account_name}, ': ', ${account_id})) AS account_name, + ${account_id} AS parent_account_id, + CAST('' AS varchar) account_status +FROM ${metadata_table_name} -WHERE +WHERE ${account_name} <> '' -GROUP BY +GROUP BY ${account_id} diff --git a/cid/builtin/core/data/queries/shared/account_map_dummy.sql b/cid/builtin/core/data/queries/shared/account_map_dummy.sql index 54e62769..8a7d10dc 100644 --- a/cid/builtin/core/data/queries/shared/account_map_dummy.sql +++ b/cid/builtin/core/data/queries/shared/account_map_dummy.sql @@ -1,10 +1,12 @@ -CREATE OR REPLACE VIEW ${athena_view_name} AS +CREATE OR REPLACE VIEW ${athena_view_name} AS SELECT DISTINCT line_item_usage_account_id account_id, - MAX_BY(bill_payer_account_id, line_item_usage_start_date) parent_account_id, MAX_BY(line_item_usage_account_id, line_item_usage_start_date) account_name, - MAX_BY(line_item_usage_account_id, line_item_usage_start_date) account_email_id + MAX_BY(bill_payer_account_id, line_item_usage_start_date) parent_account_id, + CAST('' AS varchar) account_status FROM "${cur_table_name}" +WHERE + line_item_usage_account_id <> '' GROUP BY - line_item_usage_account_id + line_item_usage_account_id \ No newline at end of file diff --git a/cid/builtin/core/data/queries/shared/account_map_org.sql b/cid/builtin/core/data/queries/shared/account_map_org.sql new file mode 100644 index 00000000..cbd7e6ff --- /dev/null +++ b/cid/builtin/core/data/queries/shared/account_map_org.sql @@ -0,0 +1,6 @@ +CREATE OR REPLACE VIEW ${athena_view_name} AS +SELECT + * +FROM + ( VALUES ${rows} ) +ignored_table_name (account_id, account_name, parent_account_id, account_status) \ No newline at end of file diff --git a/cid/builtin/core/data/queries/shared/aws_accounts.sql b/cid/builtin/core/data/queries/shared/aws_accounts.sql index 0511c7c3..90e2fd2a 100644 --- a/cid/builtin/core/data/queries/shared/aws_accounts.sql +++ b/cid/builtin/core/data/queries/shared/aws_accounts.sql @@ -1,8 +1,7 @@ CREATE OR REPLACE VIEW aws_accounts AS WITH m AS ( SELECT ${account_id} as account_id, - ${account_name} as account_name, - email account_email_id + ${account_name} as account_name FROM ${metadata_table_name} ), cur AS ( @@ -13,7 +12,6 @@ CREATE OR REPLACE VIEW aws_accounts AS WITH SELECT m.account_id, m.account_name, cur.parent_account_id, - m.account_email_id, 'Active' account_status FROM ( m diff --git a/cid/helpers/account_map.py b/cid/helpers/account_map.py index 1dd9b336..b93055da 100644 --- a/cid/helpers/account_map.py +++ b/cid/helpers/account_map.py @@ -24,11 +24,11 @@ class AccountMap(CidBase): mappings = { 'account_map': { 'acc_metadata_details': {'account_id': 'account_id', 'account_name': 'account_name'}, - 'organisation_data': {'account_id': 'id', 'account_name': 'name', 'email': 'email'} + 'organisation_data': {'account_id': 'id', 'account_name': 'name'} }, 'aws_accounts': { - 'acc_metadata_details': {'account_id': 'account_id', 'account_name': 'account_name', 'email': 'email'}, - 'organisation_data': { 'account_id': 'id', 'account_name': 'name', 'email': 'email', 'status': 'status'}, + 'acc_metadata_details': {'account_id': 'account_id', 'account_name': 'account_name'}, + 'organisation_data': { 'account_id': 'id', 'account_name': 'name', 'status': 'status'}, 'cur_fields': ['bill_payer_account_id'] } } @@ -82,13 +82,11 @@ def accounts(self) -> dict: account.update({ 'parent_account_id': account.get('parent_account_id', '0'), 'account_status': account.get('account_status', 'unknown'), - 'account_email': account.get('account_email', 'unknown') }) return self._accounts def create(self, name) -> bool: """Create account map""" - print(f'\nCreating {name}...') logger.info(f'Creating account mapping "{name}"...') try: @@ -156,10 +154,7 @@ def create(self, name) -> bool: def get_dummy_account_mapping_sql(self, name) -> list: """Create dummy account mapping""" logger.info(f'Creating dummy account mapping for {name}') - template = Template(resource_string( - package_or_requirement='cid.builtin.core', - resource_name='data/queries/shared/account_map_dummy.sql', - ).decode('utf-8')) + template = self.get_template('data/queries/shared/account_map_dummy.sql') columns_tpl = { 'athena_view_name': name, 'cur_table_name': self.cur.tableName @@ -180,8 +175,7 @@ def get_organization_accounts(self) -> list: accounts.append({ 'account_id': account.get('Id'), 'account_name': account.get('Name'), - 'account_status': account.get('Status'), - 'account_email': account.get('Email') + 'account_status': account.get('Status') }) except orgs.exceptions.AWSOrganizationsNotInUseException: print('AWS Organization is not enabled') @@ -199,6 +193,15 @@ def check_file_exists(self, file_path) -> bool: return Path.is_file(abs_path / file_path) + + def get_template(self, template_name): + """ Create template from provided template file """ + return Template(resource_string( + package_or_requirement='cid.builtin.core', + resource_name=template_name, + ).decode('utf-8')) + + def get_csv_accounts(self, file_path) -> list: """ Retreive accounts from CSV file """ with open(file_path) as f: @@ -264,20 +267,14 @@ def create_account_mapping_sql(self, name) -> str: if self._metadata_source == 'dummy': return self.get_dummy_account_mapping_sql(name) - template_str = '''CREATE OR REPLACE VIEW ${athena_view_name} AS - SELECT - * - FROM - ( VALUES ${rows} ) - ignored_table_name (account_id, account_name, parent_account_id, account_status, account_email) - ''' - template = Template(template_str) + template = self.get_template('data/queries/shared/account_map_org.sql') + accounts_sql = list() for account in self.accounts: acc = account.copy() account_name = acc.pop('account_name').replace("'", "''") accounts_sql.append( - """ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}', '{account_email}')""".format(account_name=account_name, **acc)) + """ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}')""".format(account_name=account_name, **acc)) # Fill in TPLs columns_tpl = { 'athena_view_name': name,