From e1f75335331b27af48073d1d1f3320e3e409a5b2 Mon Sep 17 00:00:00 2001 From: rosstimothy <39066650+rosstimothy@users.noreply.github.com> Date: Wed, 16 Oct 2024 23:30:42 +0000 Subject: [PATCH] Adds support to Firestore backends for custom databases (#47540) (#47584) Firestore has supported multiple databases in a project for a while (see https://cloud.google.com/blog/products/databases/manage-multiple-firestore-databases-in-a-project), however, Teleport only allowed using the default database in the project. The DatabaseID is now exposed in the file config, and if provided both the state and events backends will use the appropriate database. Closes https://github.com/gravitational/teleport/issues/37227. --- docs/pages/reference/backends.mdx | 6 +++- lib/backend/firestore/firestorebk.go | 29 ++++++++++++++----- lib/events/firestoreevents/firestoreevents.go | 8 +++-- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/docs/pages/reference/backends.mdx b/docs/pages/reference/backends.mdx index 4fad1f9b58a1..9dfc904f86f1 100644 --- a/docs/pages/reference/backends.mdx +++ b/docs/pages/reference/backends.mdx @@ -1293,6 +1293,10 @@ teleport: # Name of the Firestore table. collection_name: Example_TELEPORT_FIRESTORE_TABLE_NAME + # An optional database id to use. If not provided the default + # database for the project is used. + database_id: Example_TELEPORT_FIRESTORE_DATABASE_ID + credentials_path: /var/lib/teleport/gcs_creds # This setting configures Teleport to send the audit events to three places: @@ -1301,7 +1305,7 @@ teleport: # database table, so attempting to use the same table for both will result in errors. # When using highly available storage like Firestore, you should make sure that the list always specifies # the High Availability storage method first, as this is what the Teleport web UI uses as its source of events to display. - audit_events_uri: ['firestore://Example_TELEPORT_FIRESTORE_EVENTS_TABLE_NAME', 'file:///var/lib/teleport/audit/events', 'stdout://'] + audit_events_uri: ['firestore://Example_TELEPORT_FIRESTORE_EVENTS_TABLE_NAME?projectID=$PROJECT_ID&credentialsPath=$CREDENTIALS_PATH&databaseID=$DATABASE_ID', 'file:///var/lib/teleport/audit/events', 'stdout://'] # This setting configures Teleport to save the recorded sessions in GCP storage: audit_sessions_uri: gs://Example_TELEPORT_GCS_BUCKET/records diff --git a/lib/backend/firestore/firestorebk.go b/lib/backend/firestore/firestorebk.go index e246a996d546..7e80d9f4e4c3 100644 --- a/lib/backend/firestore/firestorebk.go +++ b/lib/backend/firestore/firestorebk.go @@ -20,6 +20,7 @@ package firestore import ( "bytes" + "cmp" "context" "encoding/base64" "errors" @@ -77,6 +78,9 @@ type Config struct { DisableExpiredDocumentPurge bool `json:"disable_expired_document_purge,omitempty"` // EndPoint is used to point the Firestore clients at emulated Firestore storage. EndPoint string `json:"endpoint,omitempty"` + // DatabaseID is the identifier of a specific Firestore database to use. If not specified, the + // default database for the ProjectID is used. + DatabaseID string `json:"database_id,omitempty"` } type backendConfig struct { @@ -320,14 +324,14 @@ func (t ownerCredentials) GetRequestMetadata(context.Context, ...string) (map[st func (t ownerCredentials) RequireTransportSecurity() bool { return false } // CreateFirestoreClients creates a firestore admin and normal client given the supplied parameters -func CreateFirestoreClients(ctx context.Context, projectID string, endPoint string, credentialsFile string) (*apiv1.FirestoreAdminClient, *firestore.Client, error) { +func CreateFirestoreClients(ctx context.Context, projectID, database string, endpoint string, credentialsFile string) (*apiv1.FirestoreAdminClient, *firestore.Client, error) { var args []option.ClientOption - if endPoint != "" { + if endpoint != "" { args = append(args, option.WithTelemetryDisabled(), option.WithoutAuthentication(), - option.WithEndpoint(endPoint), + option.WithEndpoint(endpoint), option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), option.WithGRPCDialOption(grpc.WithPerRPCCredentials(ownerCredentials{})), ) @@ -335,11 +339,21 @@ func CreateFirestoreClients(ctx context.Context, projectID string, endPoint stri args = append(args, option.WithCredentialsFile(credentialsFile)) } - firestoreClient, err := firestore.NewClient(ctx, projectID, args...) + firestoreAdminClient, err := apiv1.NewFirestoreAdminClient(ctx, args...) if err != nil { return nil, nil, ConvertGRPCError(err) } - firestoreAdminClient, err := apiv1.NewFirestoreAdminClient(ctx, args...) + + if database == "" { + firestoreClient, err := firestore.NewClient(ctx, projectID, args...) + if err != nil { + return nil, nil, ConvertGRPCError(err) + } + + return firestoreAdminClient, firestoreClient, nil + } + + firestoreClient, err := firestore.NewClientWithDatabase(ctx, projectID, database, args...) if err != nil { return nil, nil, ConvertGRPCError(err) } @@ -383,7 +397,7 @@ func New(ctx context.Context, params backend.Params, options Options) (*Backend, } closeCtx, cancel := context.WithCancel(ctx) - firestoreAdminClient, firestoreClient, err := CreateFirestoreClients(closeCtx, cfg.ProjectID, cfg.EndPoint, cfg.CredentialsPath) + firestoreAdminClient, firestoreClient, err := CreateFirestoreClients(closeCtx, cfg.ProjectID, cfg.DatabaseID, cfg.EndPoint, cfg.CredentialsPath) if err != nil { cancel() return nil, trace.Wrap(err) @@ -1095,7 +1109,8 @@ func ConvertGRPCError(err error, args ...interface{}) error { } func (b *Backend) getIndexParent() string { - return "projects/" + b.ProjectID + "/databases/(default)/collectionGroups/" + b.CollectionName + database := cmp.Or(b.backendConfig.Config.DatabaseID, "(default)") + return "projects/" + b.ProjectID + "/databases/" + database + "/collectionGroups/" + b.CollectionName } func (b *Backend) ensureIndexes(adminSvc *apiv1.FirestoreAdminClient) error { diff --git a/lib/events/firestoreevents/firestoreevents.go b/lib/events/firestoreevents/firestoreevents.go index 55d7f0a9d48d..1da33c0fc0f7 100644 --- a/lib/events/firestoreevents/firestoreevents.go +++ b/lib/events/firestoreevents/firestoreevents.go @@ -19,6 +19,7 @@ package firestoreevents import ( + "cmp" "context" "encoding/json" "errors" @@ -202,6 +203,8 @@ func (cfg *EventsConfig) SetFromURL(url *url.URL) error { } cfg.ProjectID = projectIDParamString + cfg.DatabaseID = url.Query().Get("databaseID") + eventRetentionPeriodParamString := url.Query().Get(eventRetentionPeriodPropertyKey) if eventRetentionPeriodParamString == "" { cfg.RetentionPeriod = defaultEventRetentionPeriod @@ -286,7 +289,7 @@ func New(cfg EventsConfig) (*Log, error) { }) l.Info("Initializing event backend.") closeCtx, cancel := context.WithCancel(context.Background()) - firestoreAdminClient, firestoreClient, err := firestorebk.CreateFirestoreClients(closeCtx, cfg.ProjectID, cfg.EndPoint, cfg.CredentialsPath) + firestoreAdminClient, firestoreClient, err := firestorebk.CreateFirestoreClients(closeCtx, cfg.ProjectID, cfg.DatabaseID, cfg.EndPoint, cfg.CredentialsPath) if err != nil { cancel() return nil, trace.Wrap(err) @@ -576,7 +579,8 @@ type searchEventsFilter struct { } func (l *Log) getIndexParent() string { - return "projects/" + l.ProjectID + "/databases/(default)/collectionGroups/" + l.CollectionName + database := cmp.Or(l.Config.DatabaseID, "(default)") + return "projects/" + l.ProjectID + "/databases/" + database + "/collectionGroups/" + l.CollectionName } func (l *Log) ensureIndexes(adminSvc *apiv1.FirestoreAdminClient) error {