delete joinable view in bigquery when underlying dataset(nada) is del…

…eted

pkg/access/ensurer.go

@@ -33,10 +33,12 @@ type Repo interface {
 	ListActiveAccessToDataset(ctx context.Context, datasetID uuid.UUID) ([]*models.Access, error)
 	GetOwnerGroupOfDataset(ctx context.Context, datasetID uuid.UUID) (string, error)
 	SetJoinableViewDeleted(ctx context.Context, id uuid.UUID) error
+	GetJoinableViewsToBeDeletedWithRefDatasource(ctx context.Context) ([]gensql.GetJoinableViewsToBeDeletedWithRefDatasourceRow, error)
 }
 
 type BigQuery interface {
 	DeleteJoinableDataset(ctx context.Context, datasetID string) error
+	DeleteJoinableView(ctx context.Context, joinableViewName, refProjectID, refDatasetID, refTableID string) error
 }
 
 type Revoker interface {
@@ -95,14 +97,38 @@ func (e *Ensurer) run(ctx context.Context) {
 		}
 	}
 
+	if err := e.ensureDeleteBQDatasourceForDeletedDataset(ctx); err != nil {
+		e.log.WithError(err).Error("ensuring delete bq datasource for deleted dataset")
+	}
+
 	if err := e.ensureJoinableViewAccesses(ctx); err != nil {
 		e.log.WithError(err).Error("ensuring joinable view accesses")
 	}
 }
 
+func (e *Ensurer) ensureDeleteBQDatasourceForDeletedDataset(ctx context.Context) error {
+	fmt.Println("remove deleted joinable view")
+	jvdatasources, err := e.repo.GetJoinableViewsToBeDeletedWithRefDatasource(ctx)
+	if err != nil {
+		return err
+	}
+	fmt.Println(jvdatasources)
+
+	for _, jvds := range jvdatasources {
+		err := e.bq.DeleteJoinableView(ctx, jvds.JoinableViewName, jvds.BqProjectID, jvds.BqDatasetID, jvds.BqTableID)
+		if err != nil {
+			e.log.WithError(err).Errorf("deleting joinable view with deleted pseudo-datasource %v %v.%v.%v", jvds.JoinableViewName, jvds.BqProjectID, jvds.BqDatasetID, jvds.BqTableID)
+			continue
+		}
+	}
+
+	return nil
+}
+
 func (e *Ensurer) ensureJoinableViewAccesses(ctx context.Context) error {
 	joinableViews, err := e.repo.GetJoinableViewsWithReference(ctx)
 	if err != nil {
+		e.log.WithError(err).Error("getting joinable views with reference")
 		return err
 	}
 
@@ -125,6 +151,7 @@ OUTER:
 		joinableViewName := bigquery.MakeJoinableViewName(jv.PseudoProjectID, jv.PseudoDataset, jv.PseudoTable)
 		datasetOwnerGroup, err := e.repo.GetOwnerGroupOfDataset(ctx, jv.PseudoViewID)
 		if err != nil {
+			e.log.WithError(err).Errorf("getting owner group of dataset: %v", jv.PseudoViewID)
 			return err
 		}
 		userGroups, err := e.googleGroups.Groups(ctx, &jv.Owner)
@@ -145,6 +172,7 @@ OUTER:
 
 		accesses, err := e.repo.ListActiveAccessToDataset(ctx, jv.PseudoViewID)
 		if err != nil {
+			e.log.WithError(err).Errorf("listing active access to dataset: %v", jv.PseudoViewID)
 			return err
 		}
 

pkg/access/ensurer_test.go

@@ -81,6 +81,10 @@ func (m *MockRepo) SetJoinableViewDeleted(ctx context.Context, joinableViewID uu
 	return nil
 }
 
+func (m *MockRepo) GetJoinableViewsToBeDeletedWithRefDatasource(ctx context.Context) ([]gensql.GetJoinableViewsToBeDeletedWithRefDatasourceRow, error) {
+	return nil, nil
+}
+
 type MockAM struct {
 	NGrant  int
 	NRevoke int
@@ -104,3 +108,7 @@ func (b *MockBigQuery) DeleteJoinableDataset(ctx context.Context, datasetID stri
 	b.NDeleteDataset++
 	return nil
 }
+
+func (b *MockBigQuery) DeleteJoinableView(ctx context.Context, joinableViewName, refProjectID, refDatasetID, refTableID string) error {
+	return nil
+}

pkg/bigquery/bigquery.go

@@ -421,6 +421,23 @@ func (c *Bigquery) insertSecretIfNotExists(ctx context.Context, secretDatasetID,
 	return nil
 }
 
+func (c *Bigquery) DeleteJoinableView(ctx context.Context, joinableViewName, refProjectID, refDatasetID, refTableID string) error {
+	client, err := bigquery.NewClient(ctx, c.centralDataProject)
+	if err != nil {
+		return fmt.Errorf("bigquery.NewClient: %v", err)
+	}
+	defer client.Close()
+
+	if err := client.Dataset(joinableViewName).Table(MakeJoinableViewName(refProjectID, refDatasetID, refTableID)).Delete(ctx); err != nil {
+		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
+			return nil
+		}
+		return err
+	}
+
+	return nil
+}
+
 func makeUserPrefix(user *auth.User) string {
 	emailFixDot := strings.ReplaceAll(user.Email, ".", "_")
 	emailFixAt := strings.ReplaceAll(emailFixDot, "@", "_")

pkg/bigquery/mock.go

@@ -127,3 +127,7 @@ func (c *Mock) DeleteJoinableDataset(ctx context.Context, datasetID string) erro
 func (c *Mock) MakeBigQueryUrlForJoinableViews(name, projectID, datasetID, tableID string) string {
 	return fmt.Sprintf("%v.%v.%v", "centralDataProject", name, fmt.Sprintf("%v_%v_%v", projectID, datasetID, tableID))
 }
+
+func (c *Mock) DeleteJoinableView(ctx context.Context, joinableViewName, refProjectID, refDatasetID, refTableID string) error {
+	return nil
+}

pkg/database/joinable_views.go

@@ -85,6 +85,10 @@ func (r *Repo) GetJoinableViewsWithReference(ctx context.Context) ([]gensql.GetJ
 	return r.querier.GetJoinableViewsWithReference(ctx)
 }
 
+func (r *Repo) GetJoinableViewsToBeDeletedWithRefDatasource(ctx context.Context) ([]gensql.GetJoinableViewsToBeDeletedWithRefDatasourceRow, error) {
+	return r.querier.GetJoinableViewsToBeDeletedWithRefDatasource(ctx)
+}
+
 func (r *Repo) GetJoinableViewWithAccessStatus(ctx context.Context, joinableViewID uuid.UUID, user *auth.User) (*JoinableViewInDetail, error) {
 	joinableViewDatasets, err := r.querier.GetJoinableViewWithDataset(ctx, joinableViewID)
 	if err != nil {

pkg/database/queries/joinable_views.sql

@@ -62,7 +62,7 @@ FROM
     JOIN joinable_views_datasource b ON a.id = b.joinable_view_id
     JOIN datasource_bigquery c ON b.datasource_id = c.id
 WHERE
-    a.deleted IS NULL;
+    a.deleted IS NULL AND b.deleted IS NULL;
 
 -- name: SetJoinableViewDeleted :exec
 UPDATE
@@ -98,4 +98,18 @@ FROM
     )
     INNER JOIN dataproducts dp ON datasets.dataproduct_id = dp.id
 WHERE
-    jv.id = @id;
+    jv.id = @id;
+
+-- name: GetJoinableViewsToBeDeletedWithRefDatasource :many
+SELECT
+    jv.id as joinable_view_id,
+    jv.name as joinable_view_name,
+    bq.project_id as bq_project_id,
+    bq.dataset as bq_dataset_id,
+    bq.table_name as bq_table_id
+FROM
+    joinable_views jv
+    JOIN joinable_views_datasource jvds ON jv.id = jvds.joinable_view_id
+    JOIN datasource_bigquery bq ON bq.id = jvds.datasource_id
+WHERE
+    jvds.deleted IS NOT NULL;

pkg/graph/resolver.go

@@ -27,6 +27,7 @@ type Bigquery interface {
 	CreateJoinableViewsForUser(ctx context.Context, name string, datasources []bq.JoinableViewDatasource) (string, string, map[uuid.UUID]string, error)
 	MakeBigQueryUrlForJoinableViews(name, projectID, datasetID, tableID string) string
 	DeleteJoinableDataset(ctx context.Context, datasetID string) error
+	DeleteJoinableView(ctx context.Context, joinableViewName, refProjectID, refDatasetID, refTableID string) error
 }
 
 type AccessManager interface {