delete pseudo view when dataset is deleted

pkg/access/ensurer.go

@@ -34,11 +34,14 @@ type Repo interface {
 	GetOwnerGroupOfDataset(ctx context.Context, datasetID uuid.UUID) (string, error)
 	SetJoinableViewDeleted(ctx context.Context, id uuid.UUID) error
 	GetJoinableViewsToBeDeletedWithRefDatasource(ctx context.Context) ([]gensql.GetJoinableViewsToBeDeletedWithRefDatasourceRow, error)
+	GetPseudoDatasourcesToDelete(ctx context.Context) ([]*models.BigQuery, error)
+	SetDatasourceDeleted(ctx context.Context, id uuid.UUID) error
 }
 
 type BigQuery interface {
 	DeleteJoinableDataset(ctx context.Context, datasetID string) error
 	DeleteJoinableView(ctx context.Context, joinableViewName, refProjectID, refDatasetID, refTableID string) error
+	DeletePseudoView(ctx context.Context, pseudoProjectID, pseudoDatasetID, pseudoTableID string) error
 }
 
 type Revoker interface {
@@ -97,22 +100,56 @@ func (e *Ensurer) run(ctx context.Context) {
 		}
 	}
 
-	if err := e.ensureDeleteBQDatasourceForDeletedDataset(ctx); err != nil {
+	if err := e.ensureDeleteJoinableViewBQForDeletedDataset(ctx); err != nil {
 		e.log.WithError(err).Error("ensuring delete bq datasource for deleted dataset")
 	}
 
 	if err := e.ensureJoinableViewAccesses(ctx); err != nil {
 		e.log.WithError(err).Error("ensuring joinable view accesses")
 	}
+
+	if err := e.ensureDeletePseudoViewBQForDeletedDataset(ctx); err != nil {
+		e.log.WithError(err).Error("ensuring delete pseudo view for deleted dataset")
+	}
+}
+
+func (e *Ensurer) ensureDeletePseudoViewBQForDeletedDataset(ctx context.Context) error {
+	pseudoDatasources, err := e.repo.GetPseudoDatasourcesToDelete(ctx)
+	if err != nil {
+		return err
+	}
+
+	if len(pseudoDatasources) == 0 {
+		return nil
+	}
+
+	e.log.Infof("Delete pseudo views without a dataset: %v", pseudoDatasources)
+
+	for _, pds := range pseudoDatasources {
+		if len(pds.PseudoColumns) == 0 {
+			e.log.Errorf("deleting pseudo view without pseudo columns, ignored")
+			continue
+		}
+
+		if err := e.bq.DeletePseudoView(ctx, pds.ProjectID, pds.Dataset, pds.Table); err != nil {
+			e.log.WithError(err).Errorf("deleting pseudo view with deleted dataset %v", pds.Dataset)
+			continue
+		}
+
+		if err := e.repo.SetDatasourceDeleted(ctx, pds.ID); err != nil {
+			e.log.WithError(err).Errorf("setting pseudo view deleted in db, view id: %v", pds.ID)
+		} else {
+			e.log.Infof("pseudo view without dataset deleted: %v", pds.ID)
+		}
+	}
+	return nil
 }
 
-func (e *Ensurer) ensureDeleteBQDatasourceForDeletedDataset(ctx context.Context) error {
-	fmt.Println("remove deleted joinable view")
+func (e *Ensurer) ensureDeleteJoinableViewBQForDeletedDataset(ctx context.Context) error {
 	jvdatasources, err := e.repo.GetJoinableViewsToBeDeletedWithRefDatasource(ctx)
 	if err != nil {
 		return err
 	}
-	fmt.Println(jvdatasources)
 
 	for _, jvds := range jvdatasources {
 		err := e.bq.DeleteJoinableView(ctx, jvds.JoinableViewName, jvds.BqProjectID, jvds.BqDatasetID, jvds.BqTableID)

pkg/access/ensurer_test.go

@@ -81,6 +81,14 @@ func (m *MockRepo) SetJoinableViewDeleted(ctx context.Context, joinableViewID uu
 	return nil
 }
 
+func (m *MockRepo) GetPseudoDatasourcesToDelete(ctx context.Context) ([]*models.BigQuery, error) {
+	return nil, nil
+}
+
+func (m *MockRepo) SetDatasourceDeleted(ctx context.Context, id uuid.UUID) error {
+	return nil
+}
+
 func (m *MockRepo) GetJoinableViewsToBeDeletedWithRefDatasource(ctx context.Context) ([]gensql.GetJoinableViewsToBeDeletedWithRefDatasourceRow, error) {
 	return nil, nil
 }
@@ -112,3 +120,7 @@ func (b *MockBigQuery) DeleteJoinableDataset(ctx context.Context, datasetID stri
 func (b *MockBigQuery) DeleteJoinableView(ctx context.Context, joinableViewName, refProjectID, refDatasetID, refTableID string) error {
 	return nil
 }
+
+func (b *MockBigQuery) DeletePseudoView(ctx context.Context, pseudoProjectID, pseudoDatasetID, pseudoTableID string) error {
+	return nil
+}

pkg/bigquery/bigquery.go

@@ -422,13 +422,24 @@ func (c *Bigquery) insertSecretIfNotExists(ctx context.Context, secretDatasetID,
 }
 
 func (c *Bigquery) DeleteJoinableView(ctx context.Context, joinableViewName, refProjectID, refDatasetID, refTableID string) error {
-	client, err := bigquery.NewClient(ctx, c.centralDataProject)
+	return c.deleteBigqueryTable(ctx, c.centralDataProject, joinableViewName, MakeJoinableViewName(refProjectID, refDatasetID, refTableID))
+}
+
+func (c *Bigquery) DeletePseudoView(ctx context.Context, pseudoProjectID, pseudoDatasetID, pseudoTableID string) error {
+	if pseudoDatasetID != c.pseudoDataset {
+		return fmt.Errorf("cannot delete pseudo view from dataset %v, not a markedsplassen dataset", pseudoDatasetID)
+	}
+	return c.deleteBigqueryTable(ctx, pseudoProjectID, pseudoDatasetID, pseudoTableID)
+}
+
+func (c *Bigquery) deleteBigqueryTable(ctx context.Context, projectID, datasetID, tableID string) error {
+	client, err := bigquery.NewClient(ctx, projectID)
 	if err != nil {
 		return fmt.Errorf("bigquery.NewClient: %v", err)
 	}
 	defer client.Close()
 
-	if err := client.Dataset(joinableViewName).Table(MakeJoinableViewName(refProjectID, refDatasetID, refTableID)).Delete(ctx); err != nil {
+	if err := client.Dataset(datasetID).Table(tableID).Delete(ctx); err != nil {
 		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
 			return nil
 		}

pkg/bigquery/mock.go

@@ -131,3 +131,7 @@ func (c *Mock) MakeBigQueryUrlForJoinableViews(name, projectID, datasetID, table
 func (c *Mock) DeleteJoinableView(ctx context.Context, joinableViewName, refProjectID, refDatasetID, refTableID string) error {
 	return nil
 }
+
+func (c *Mock) DeletePseudoView(ctx context.Context, pseudoProjectID, pseudoDatasetID, pseudoTableID string) error {
+	return nil
+}

pkg/database/datasets.go

@@ -239,7 +239,6 @@ func (r *Repo) UpdateDataset(ctx context.Context, id uuid.UUID, new models.Updat
 	return datasetFromSQL(res), nil
 }
 
-
 func (r *Repo) GetBigqueryDatasource(ctx context.Context, datasetID uuid.UUID, isReference bool) (models.BigQuery, error) {
 	bq, err := r.querier.GetBigqueryDatasource(ctx, gensql.GetBigqueryDatasourceParams{
 		DatasetID:   datasetID,
@@ -410,7 +409,28 @@ func (r *Repo) GetAccessiblePseudoDatasourcesByUser(ctx context.Context, subject
 	return pseudoDatasets, nil
 }
 
+func (r *Repo) GetPseudoDatasourcesToDelete(ctx context.Context) ([]*models.BigQuery, error) {
+	rows, err := r.querier.GetPseudoDatasourcesToDelete(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	pseudoViews := []*models.BigQuery{}
+	for _, d := range rows {
+		pseudoViews = append(pseudoViews, &models.BigQuery{
+			ID:            d.ID,
+			Dataset:       d.Dataset,
+			ProjectID:     d.ProjectID,
+			Table:         d.TableName,
+			PseudoColumns: d.PseudoColumns,
+		})
+	}
+	return pseudoViews, nil
+}
 
+func (r *Repo) SetDatasourceDeleted(ctx context.Context, id uuid.UUID) error {
+	return r.querier.SetDatasourceDeleted(ctx, id)
+}
 
 func (r *Repo) GetOwnerGroupOfDataset(ctx context.Context, datasetID uuid.UUID) (string, error) {
 	return r.querier.GetOwnerGroupOfDataset(ctx, datasetID)

pkg/database/migrations/0077_add_deleted_to_datasource.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+ALTER TABLE datasource_bigquery ADD COLUMN "deleted" TIMESTAMPTZ;
+
+-- +goose Down
+ALTER TABLE datasource_bigquery DROP COLUMN "deleted";

pkg/database/queries/datasets.sql

@@ -301,4 +301,23 @@ WHERE
       dataproducts dp
     WHERE
       dp.group = ANY(@groups :: text [])
-  );
+  );
+
+-- name: GetPseudoDatasourcesToDelete :many
+SELECT
+  bq.*
+FROM
+  datasource_bigquery bq
+  LEFT JOIN datasets ds ON bq.dataset_id = ds.id
+WHERE
+  ds.id IS NULL
+  AND bq.deleted is NULL
+  AND ARRAY_LENGTH(bq.pseudo_columns, 1) > 0;
+
+-- name: SetDatasourceDeleted :exec
+UPDATE
+  datasource_bigquery
+SET
+  deleted = NOW()
+WHERE
+  id = @id;

pkg/graph/resolver.go

@@ -28,6 +28,7 @@ type Bigquery interface {
 	MakeBigQueryUrlForJoinableViews(name, projectID, datasetID, tableID string) string
 	DeleteJoinableDataset(ctx context.Context, datasetID string) error
 	DeleteJoinableView(ctx context.Context, joinableViewName, refProjectID, refDatasetID, refTableID string) error
+	DeletePseudoView(ctx context.Context, pseudoProjectID, pseudoDatasetID, pseudoTableID string) error
 }
 
 type AccessManager interface {