use biquery fetch endpoint

Author: MengZhuNAV

components/dataproducts/dataset/useColumnTags.tsx

@@ -1,8 +1,8 @@
 import { useState } from 'react'
 import {
   BigQueryType,
-  useGcpGetColumnsQuery,
 } from '../../../lib/schema/graphql'
+import { useFetchBQcolumns } from '../../../lib/rest/bigquery'
 
 export type PIITagType =
   | 'PII_DirekteIdentifiserende'
@@ -52,14 +52,7 @@ export const useColumnTags = (
   const [pseudoColumnsMap, setPseudoColumnsMap] = useState<PseudoColumnsMapType>(
     new Map<string, Map<string, boolean>>()
   )
-
-  const columnsQuery = useGcpGetColumnsQuery({
-    variables: {
-      projectID: projectID,
-      datasetID: datasetID,
-      tableID: tableID,
-    },
-  })
+  const fetchColumns = useFetchBQcolumns(projectID, datasetID, tableID)
 
   var tableKey = buildTableKey(projectID, datasetID, tableID)
 
@@ -69,9 +62,9 @@ export const useColumnTags = (
       datasetID &&
       tableID &&
       !tagsMap.has(tableKey) &&
-      !columnsQuery.error &&
-      !columnsQuery.loading &&
-      columnsQuery.data
+      !fetchColumns.error &&
+      !fetchColumns.loading &&
+      fetchColumns.bqColumns
     ) {
       var newTagsMap = new Map<string, Map<string, PIITagType> | undefined>(
         tagsMap
@@ -81,7 +74,7 @@ export const useColumnTags = (
 
       var newPseudoColumnsMap = new Map<string, Map<string, boolean>>(pseudoColumnsMap)
       var pseudoColumns = new Map<string, boolean>()
-      columnsQuery.data.gcpGetColumns.forEach((it) =>{
+      fetchColumns.bqColumns.forEach((it) =>{
         tags.set(
           it.name,
           (!!tagsFromQuery[it.name] &&
@@ -126,11 +119,11 @@ export const useColumnTags = (
 
   return {
     columns:
-      !columnsQuery.error && !columnsQuery.loading
-        ? (columnsQuery.data?.gcpGetColumns as ColumnType[])
+      !fetchColumns.error && !fetchColumns.loading
+        ? (fetchColumns.bqColumns as ColumnType[])
         : undefined,
-    loading: columnsQuery.loading,
-    error: columnsQuery.error,
+    loading: fetchColumns.loading,
+    error: fetchColumns.error,
     tags: tagsMap.get(tableKey),
     pseudoColumns: pseudoColumnsMap.get(tableKey) || new Map<string, boolean>(),
     annotateColumn: annotateColumn,

components/dataproducts/datasource/dataset.tsx

@@ -1,19 +1,18 @@
 import {
   BigQueryType,
-  GcpGetTablesQuery,
-  useGcpGetTablesLazyQuery,
 } from '../../../lib/schema/graphql'
 
 import { TreeItem } from '@mui/x-tree-view/TreeItem';
 
 import { Loader } from '@navikt/ds-react'
 import { ExpandFilled, NextFilled } from '@navikt/ds-icons'
 import Tabell from '../../lib/icons/tabell'
+import { useFetchBQTables } from '../../../lib/rest/bigquery';
 
-const DataproductTableIconMap: Record<BigQueryType, () => JSX.Element> = {
-  materialized_view: Tabell,
-  table: Tabell,
-  view: Tabell,
+const DataproductTableIconMap: Record<string, () => JSX.Element> = {
+  "materialized_view": Tabell,
+  "table": Tabell,
+  "view": Tabell,
 }
 
 export interface DataproductSourceDatasetProps {
@@ -27,11 +26,7 @@ export const Dataset = ({
   datasetID,
   active,
 }: DataproductSourceDatasetProps) => {
-  const [getTables, { data, loading, called }] = useGcpGetTablesLazyQuery({
-    variables: { projectID, datasetID },
-  })
-
-  if (active && !called) getTables()
+  const fetchBQTables= useFetchBQTables(projectID, datasetID)
 
   const loadingPlaceholder = (
     <TreeItem
@@ -48,14 +43,14 @@ export const Dataset = ({
     />
   )
 
-  const datasetContents = (contents: GcpGetTablesQuery['gcpGetTables']) =>
-    contents?.map(({ name, type }) => (
+  const datasetContents = (contents: any) =>
+    contents?.map((it: any) => (
       <TreeItem
         className="MuiTreeView-leaf"
-        slots={{ endIcon: DataproductTableIconMap[type]}}
-        itemId={`${projectID}/${datasetID}/${name}`}
-        key={`${projectID}/${datasetID}/${name}`}
-        label={name}
+        slots={{ endIcon: DataproductTableIconMap[it.type as string]}}
+        itemId={`${projectID}/${datasetID}/${it.name}`}
+        key={`${projectID}/${datasetID}/${it.name}`}
+        label={it.name}
       />
     ))
 
@@ -65,10 +60,10 @@ export const Dataset = ({
       itemId={`${projectID}/${datasetID}`}
       label={datasetID}
     >
-      {loading
+      {fetchBQTables.loading
         ? loadingPlaceholder
-        : data?.gcpGetTables?.length
-        ? datasetContents(data?.gcpGetTables)
+        : fetchBQTables.bqTables?.length
+        ? datasetContents(fetchBQTables?.bqTables)
         : emptyPlaceholder}
     </TreeItem>
   )

components/dataproducts/datasource/project.tsx

@@ -1,8 +1,8 @@
 import { TreeItem } from '@mui/x-tree-view/TreeItem';
 import { Loader } from '@navikt/ds-react'
-import { useGcpGetDatasetsLazyQuery } from '../../../lib/schema/graphql'
 import { ExpandFilled, NextFilled } from '@navikt/ds-icons'
 import { Dataset } from './dataset'
+import { useFetchBQDatasets } from '../../../lib/rest/bigquery';
 
 export interface DataproductSourceDatasetProps {
   activePaths: string[]
@@ -13,12 +13,7 @@ export const Project = ({
   projectID,
   activePaths,
 }: DataproductSourceDatasetProps) => {
-  const [getDatasets, { data, loading, error, called }] =
-    useGcpGetDatasetsLazyQuery({
-      variables: { projectID },
-    })
-
-  if (!called && activePaths.includes(projectID)) getDatasets()
+  const fetchBQDatasets= useFetchBQDatasets(projectID)
 
   const emptyPlaceholder = (
     <TreeItem
@@ -41,11 +36,11 @@ export const Project = ({
       itemId={projectID}
       label={projectID}
     >
-      {loading
+      {fetchBQDatasets.loading
         ? loadingPlaceholder
-        : !data?.gcpGetDatasets?.length
+        : !fetchBQDatasets.bqDatasets?.length
         ? emptyPlaceholder
-        : data?.gcpGetDatasets.map((datasetID) => (
+        : fetchBQDatasets.bqDatasets?.map((datasetID) => (
             <Dataset
               key={datasetID}
               projectID={projectID}

components/dataproducts/datasource/projectTables.tsx

@@ -0,0 +1,86 @@
+import { useEffect, useState } from "react"
+import { fetchBQColumnsUrl, fetchBQDatasetsUrl, fetchBQTablesUrl, fetchTemplate } from "./restApi"
+
+export const fetchBQDatasets = async (projectID: string) => {
+    const url = fetchBQDatasetsUrl(projectID)
+    return fetchTemplate(url)
+}
+
+export const fetchBQTables = async (projectID: string, datasetID: string) => {
+    const url = fetchBQTablesUrl(projectID, datasetID)
+    return fetchTemplate(url)
+}
+
+export const fetchBQColumns = async (projectID: string, datasetID: string, tableID: string) => {
+    const url = fetchBQColumnsUrl(projectID, datasetID, tableID)
+    return fetchTemplate(url)
+}
+
+export const useFetchBQDatasets = (projectID: string) => {
+    const [datasets, setDatasets] = useState<any[]>([])
+    const [loading, setLoading] = useState(false)
+    const [error, setError] = useState<any>(null)
+
+    useEffect(() => {
+        setLoading(true)
+        fetchBQDatasets(projectID).then((res) => res.json())
+            .then((data) => {
+                setError(null)
+                setDatasets(data.bqDatasets)
+            })
+            .catch((err) => {
+                setError(err)
+                setDatasets([])
+            }).finally(() => {
+                setLoading(false)
+            })
+    }, [projectID])
+
+    return { bqDatasets: datasets, loading, error }
+}
+
+export const useFetchBQTables = (projectID: string, datasetID: string) => {
+    const [tables, setTables] = useState<any[]>([])
+    const [loading, setLoading] = useState(false)
+    const [error, setError] = useState<any>(null)
+
+    useEffect(() => {
+        setLoading(true)
+        fetchBQTables(projectID, datasetID).then((res) => res.json())
+            .then((data) => {
+                setError(null)
+                setTables(data.bqTables)
+            })
+            .catch((err) => {
+                setError(err)
+                setTables([])
+            }).finally(() => {
+                setLoading(false)
+            })
+    }, [projectID, datasetID])
+
+    return { bqTables: tables, loading, error }
+}
+
+export const useFetchBQcolumns = (projectID: string, datasetID: string, tableID: string) => {
+    const [columns, setColumns] = useState<any[]>([])
+    const [loading, setLoading] = useState(false)
+    const [error, setError] = useState<any>(null)
+
+    useEffect(() => {
+        setLoading(true)
+        fetchBQColumns(projectID, datasetID, tableID).then((res) => res.json())
+            .then((data) => {
+                setError(null)
+                setColumns(data.bqColumns)
+            })
+            .catch((err) => {
+                setError(err)
+                setColumns([])
+            }).finally(() => {
+                setLoading(false)
+            })
+    }, [projectID, datasetID, tableID])
+
+    return { bqColumns: columns, loading, error }
+}

lib/queries/dataproduct/dataproductSummary.ts

@@ -16,6 +16,9 @@ export const getProductAreaUrl = (id: string) => `${apiUrl()}/productareas/${id}
 export const fetchUserDataUrl = () => `${apiUrl()}/userData`
 export const fetchKeywordsUrl = () => `${apiUrl()}/keywords`
 export const fetchAccessRequestUrl = (datasetId: string) => `${apiUrl()}/accessRequests?datasetId=${datasetId}`
+export const fetchBQDatasetsUrl = (projectId: string) => `${apiUrl()}/bigquery/datasets?projectId=${projectId}`
+export const fetchBQTablesUrl = (projectId: string, datasetId: string) => `${apiUrl()}/bigquery/tables?projectId=${projectId}&datasetId=${datasetId}`
+export const fetchBQColumnsUrl = (projectId: string, datasetId: string, tableId: string) => `${apiUrl()}/bigquery/tables?projectId=${projectId}&datasetId=${datasetId}&tableId=${tableId}`
 export const searchTeamKatalogenUrl = (gcpGroups?: string[]) => {
   const parameters = gcpGroups?.length ? gcpGroups.map(group => `gcpGroups=${encodeURIComponent(group)}`).join('&') : ''
   const query = parameters ? `?${parameters}` : ''