From c0e767e9e199b0ea651ba17fbb758b4c215ce392 Mon Sep 17 00:00:00 2001 From: "mariano.pizarro" Date: Mon, 18 Apr 2022 12:39:59 -0300 Subject: [PATCH] fix: Ensure that all currently supported GCP services allow querygcpTag --- package.json | 2 +- src/services/base/schema.graphql | 12 +++++ src/services/organization/data.ts | 65 +++++++++++++++++++++++- src/services/organization/format.ts | 4 +- src/services/organization/schema.graphql | 1 + src/services/tag/schema.graphql | 2 - src/types/generated.ts | 9 +++- src/types/index.ts | 16 +++--- src/utils/format.ts | 13 ++++- yarn.lock | 8 +-- 10 files changed, 111 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index 9cbe0ea..14cf5f2 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "@google-cloud/logging": "^9.6.4", "@google-cloud/monitoring": "^2.3.5", "@google-cloud/notebooks": "^1.3.1", - "@google-cloud/resource-manager": "^3.0.0", + "@google-cloud/resource-manager": "^3.2.0", "@google-cloud/secret-manager": "^3.10.1", "@google-cloud/storage": "^5.16.1", "@google-cloud/vpc-access": "^1.1.2", diff --git a/src/services/base/schema.graphql b/src/services/base/schema.graphql index 73d293f..0a61532 100644 --- a/src/services/base/schema.graphql +++ b/src/services/base/schema.graphql @@ -22,4 +22,16 @@ type gcpKeyValue id: String! @id @search(by: [hash]) key: String! @search(by: [hash, regexp]) value: String @search(by: [hash, regexp]) + } + + type gcpKeyValues + @generate( + query: { get: true, query: true, aggregate: true } + mutation: { add: true, delete: false } + subscription: false + ) + @key(fields: "id") { + id: String! @id @search(by: [hash]) + key: String! @search(by: [hash, regexp]) + values: [String] @search(by: [hash, regexp]) } \ No newline at end of file diff --git a/src/services/organization/data.ts b/src/services/organization/data.ts index d978391..cff2dc5 100644 --- a/src/services/organization/data.ts +++ b/src/services/organization/data.ts @@ -1,9 +1,13 @@ -import { OrganizationsClient } from '@google-cloud/resource-manager' +import { + OrganizationsClient, + TagKeysClient, + TagValuesClient, +} from '@google-cloud/resource-manager' import { google } from '@google-cloud/resource-manager/build/protos/protos' import CloudGraph from '@cloudgraph/sdk' import groupBy from 'lodash/groupBy' import gcpLoggerText from '../../properties/logger' -import { GcpServiceInput } from '../../types' +import { GcpServiceInput, TagMap } from '../../types' import { initTestEndpoint, generateGcpErrorLog } from '../../utils' import { GLOBAL_REGION } from '../../config/constants' @@ -17,6 +21,7 @@ export interface RawGcpOrganization id: string projectId: string region: string + tags?: TagMap } export const listOrganizationsData = async ( @@ -52,6 +57,42 @@ export const listOrganizationsData = async ( resolve() }) +export const getTags = async ({ + tagKeysClient, + tagValuesClient, + resourceId, +}: { + tagKeysClient: TagKeysClient + tagValuesClient: TagValuesClient + resourceId: string +}): Promise => + new Promise(async resolve => { + const tags: TagMap = {} + try { + const iterable = tagKeysClient.listTagKeysAsync({ parent: resourceId }) + for await (const response of iterable) { + if (response) { + const { name: parent, shortName: tagKey } = response + const tagValuesIterable = await tagValuesClient.listTagValuesAsync({ parent }) + const tagValues: string[] = [] + for await (const tagValue of tagValuesIterable) { + if (tagValue) { + tagValues.push(tagValue.shortName) + } + } + tags[tagKey] = tagValues + } + } + } catch (error) { + generateGcpErrorLog( + serviceName, + 'resourceManager:listTagKeysAsync', + error + ) + } + resolve(tags) + }) + export default async ({ config, }: GcpServiceInput): Promise<{ @@ -59,6 +100,9 @@ export default async ({ }> => new Promise(async resolve => { const orgList: RawGcpOrganization[] = [] + const tagKeysClient = new TagKeysClient({ ...config, apiEndpoint }) + const tagValuesClient = new TagValuesClient({ ...config, apiEndpoint }) + const tagsPromises = [] const { projectId } = config const organizationsClient = new OrganizationsClient({ @@ -66,8 +110,25 @@ export default async ({ apiEndpoint, }) + // Get all Organizations await listOrganizationsData(organizationsClient, projectId, orgList) + // Add tags to each Organization + orgList.map(({ id }, idx) => { + const tagsPromise = new Promise(async resolveTags => { + const tags = await getTags({ + tagKeysClient, + tagValuesClient, + resourceId: id, + }) + + orgList[idx].tags = tags || {} + resolveTags() + }) + tagsPromises.push(tagsPromise) + }) + await Promise.all(tagsPromises) + logger.debug(lt.foundResources(serviceName, orgList.length)) resolve(groupBy(orgList, 'region')) }) diff --git a/src/services/organization/format.ts b/src/services/organization/format.ts index 274fa89..53b141f 100644 --- a/src/services/organization/format.ts +++ b/src/services/organization/format.ts @@ -2,7 +2,7 @@ import { google } from '@google-cloud/resource-manager/build/protos/protos' import { GcpOrganization } from '../../types/generated' import { RawGcpOrganization } from './data' import { toISOString } from '../../utils/dateutils' -import { enumKeyToString } from '../../utils/format' +import { enumKeyToString, formatKeyValuesFromMap } from '../../utils/format' export default ({ service, @@ -22,6 +22,7 @@ export default ({ updateTime, deleteTime, etag, + tags = {}, } = service return { @@ -36,5 +37,6 @@ export default ({ updateTime: toISOString(updateTime?.seconds?.toString()), deleteTime: toISOString(deleteTime?.seconds?.toString()), etag, + tags: formatKeyValuesFromMap(tags), } } diff --git a/src/services/organization/schema.graphql b/src/services/organization/schema.graphql index efcd6af..22ec462 100644 --- a/src/services/organization/schema.graphql +++ b/src/services/organization/schema.graphql @@ -6,6 +6,7 @@ type gcpOrganization implements gcpBaseResource @key(fields: "id") { updateTime: String @search(by: [hash, regexp]) deleteTime: String @search(by: [hash, regexp]) etag: String @search(by: [hash, regexp]) + tags: [gcpKeyValues] folders: [gcpFolder] @hasInverse(field: organization) projects: [gcpProject] @hasInverse(field: organization) } \ No newline at end of file diff --git a/src/services/tag/schema.graphql b/src/services/tag/schema.graphql index 2bfcc4a..655b40e 100644 --- a/src/services/tag/schema.graphql +++ b/src/services/tag/schema.graphql @@ -2,9 +2,7 @@ type gcpTag implements gcpBaseResource @key(fields: "id") { id: String! @id @search(by: [hash]) key: String! @search(by: [hash, regexp]) value: String! @search(by: [hash, regexp]) - folder: [gcpFolder] organization: [gcpOrganization] - project: [gcpProject] vmInstance: [gcpVmInstance] } diff --git a/src/types/generated.ts b/src/types/generated.ts index 7681da7..2a62123 100644 --- a/src/types/generated.ts +++ b/src/types/generated.ts @@ -1933,6 +1933,12 @@ export type GcpKeyValue = { value?: Maybe; }; +export type GcpKeyValues = { + id: Scalars['String']; + key: Scalars['String']; + values?: Maybe>>; +}; + export type GcpKmsCryptoKey = GcpBaseResource & { aiPlatformNotebooks?: Maybe>>; createTime?: Maybe; @@ -2179,6 +2185,7 @@ export type GcpOrganization = GcpBaseResource & { folders?: Maybe>>; projects?: Maybe>>; state?: Maybe; + tags?: Maybe>>; updateTime?: Maybe; }; @@ -2668,11 +2675,9 @@ export type GcpSubnetSecondaryRange = { }; export type GcpTag = GcpBaseResource & { - folder?: Maybe>>; id: Scalars['String']; key: Scalars['String']; organization?: Maybe>>; - project?: Maybe>>; value: Scalars['String']; vmInstance?: Maybe>>; }; diff --git a/src/types/index.ts b/src/types/index.ts index e24cdd1..78f14c8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -4,7 +4,7 @@ export interface Tags { } export interface TagMap { - [property: string]: string + [property: string]: string | string [] } export interface LabelMap { @@ -12,13 +12,7 @@ export interface LabelMap { } export interface KeyValueMapMap { - [k: string]: string | number | boolean | Long -} - -export interface GcpServiceInput { - regions: string - config: GcpCredentials - rawData: rawDataInterface[] + [k: string]: string | number | boolean | Long | string[] } export interface GcpCredentials { @@ -38,3 +32,9 @@ export interface rawDataInterface { subnet?: string[] vpcConnector?: string[] } + +export interface GcpServiceInput { + regions: string + config: GcpCredentials + rawData: rawDataInterface[] +} \ No newline at end of file diff --git a/src/utils/format.ts b/src/utils/format.ts index 9fa3d6b..9c91fa0 100644 --- a/src/utils/format.ts +++ b/src/utils/format.ts @@ -1,6 +1,6 @@ import cuid from 'cuid' import { google } from '@google-cloud/bigquery-data-transfer/build/protos/protos' -import { GcpKeyValue, GcpRawLabel, GcpRawTag, GcpBigQueryDataTransferParam } from '../types/generated' +import { GcpKeyValue, GcpKeyValues, GcpRawLabel, GcpRawTag, GcpBigQueryDataTransferParam } from '../types/generated' import { TagMap, LabelMap, KeyValueMapMap } from '../types' export const formatKeyValueMap = (keyValueMap: KeyValueMapMap): GcpKeyValue[] => { @@ -11,12 +11,23 @@ export const formatKeyValueMap = (keyValueMap: KeyValueMapMap): GcpKeyValue[] => })) } +export const formatKeyValuesMap = (keyValueMap: KeyValueMapMap): GcpKeyValues[] => { + return Object.keys(keyValueMap || {}).map(key => ({ + id: cuid(), + key, + values: keyValueMap[key] as string[] + })) +} + // We need an id here to enfore uniqueness for Dgraph, otherwise we get duplicate tags export const formatLabelsFromMap = (labels: LabelMap): GcpRawLabel[] => formatKeyValueMap(labels) // We need an id here to enfore uniqueness for Dgraph, otherwise we get duplicate tags export const formatTagsFromMap = (tags: TagMap): GcpRawTag[] => formatKeyValueMap(tags) +// We need an id here to enfore uniqueness for Dgraph, otherwise we get duplicate tags +export const formatKeyValuesFromMap = (tags: TagMap): GcpKeyValues[] => formatKeyValuesMap(tags) + export const obfuscateSensitiveString = (s: string): string => { const stars = '*'.repeat(Math.min(30, s.length - 6)) return s.slice(0, 3) + stars + s.slice(stars.length + 3, s.length) diff --git a/yarn.lock b/yarn.lock index 7f280c9..c76283b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -897,10 +897,10 @@ resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-2.0.4.tgz#9d8705ecb2baa41b6b2673f3a8e9b7b7e1abc52a" integrity sha512-j8yRSSqswWi1QqUGKVEKOG03Q7qOoZP6/h2zN2YO+F5h2+DHU0bSrHCK9Y7lo2DI9fBd8qGAw795sf+3Jva4yA== -"@google-cloud/resource-manager@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@google-cloud/resource-manager/-/resource-manager-3.0.0.tgz#4e81309e3523a410d61eda37d86884b53303bb5a" - integrity sha512-gumfF1JiglfnvhRPhpjjc9JbfcdNhcSKSbpYxAMUpFkjiPYd5z01NzEpiSwB/ucT8nUuFCqUrCHVel6gPLXk2g== +"@google-cloud/resource-manager@^3.2.0": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@google-cloud/resource-manager/-/resource-manager-3.2.0.tgz#dd0ad3e7efd591d56e06f4318fb1e4798a66fff2" + integrity sha512-5LDQ/l7QY7+TCHQzeK8+MKTwt79IxYb5hO5m3v0fd8b+VLBZhKp5bK9usluZZgdnyY7d0zIWsA1MiKtEHEzusA== dependencies: google-gax "^2.24.1"