From 535a1018a5c6656b05518713a47304018b395a07 Mon Sep 17 00:00:00 2001 From: Thomas Gerber Date: Fri, 28 Oct 2022 10:09:49 -0700 Subject: [PATCH] Trigger metabase db sync on data sync to populate filters (#242) * Trigger metabase db sync on data sync to populate filters * Inject metabase credentials from .env file Co-authored-by: Yandry Perez Clemente <99700024+ypc-faros@users.noreply.github.com> --- cli/src/airbyte/airbyte-client.ts | 2 +- cli/src/bitbucket/run.ts | 18 ++++++- cli/src/cli.ts | 27 ++++++++-- cli/src/github/run.ts | 17 ++++++- cli/src/gitlab/run.ts | 17 ++++++- cli/src/jira/run.ts | 17 ++++++- cli/src/metabase/metabase-client.ts | 56 +++++++++++++++++++++ init/resources/metabase/dashboards/git.json | 13 ++++- start.sh | 7 ++- 9 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 cli/src/metabase/metabase-client.ts diff --git a/cli/src/airbyte/airbyte-client.ts b/cli/src/airbyte/airbyte-client.ts index 99b2b7ce..faa48148 100644 --- a/cli/src/airbyte/airbyte-client.ts +++ b/cli/src/airbyte/airbyte-client.ts @@ -1,6 +1,6 @@ import retry from 'async-retry'; import axios, {AxiosInstance} from 'axios'; -import { ceil } from 'lodash'; +import {ceil} from 'lodash'; import ProgressBar from 'progress'; import {VError} from 'verror'; diff --git a/cli/src/bitbucket/run.ts b/cli/src/bitbucket/run.ts index 2dad84aa..65fc4922 100644 --- a/cli/src/bitbucket/run.ts +++ b/cli/src/bitbucket/run.ts @@ -5,6 +5,7 @@ import VError from 'verror'; import {Airbyte} from '../airbyte/airbyte-client'; import {wrapApiError} from '../cli'; +import {Metabase} from '../metabase/metabase-client'; import { display, Emoji, @@ -27,6 +28,7 @@ const DEFAULT_API_URL = 'https://api.bitbucket.org/2.0'; interface BitbucketConfig { readonly airbyte: Airbyte; + readonly metabase: Metabase; readonly serverUrl?: string; readonly username?: string; readonly password?: string; @@ -62,8 +64,12 @@ export function makeBitbucketCommand(): Command { cmd.action(async (options) => { const airbyte = new Airbyte(options.airbyteUrl); - - await runBitbucket({...options, airbyte}); + const metabase = await Metabase.fromConfig({ + url: options.metabaseUrl, + username: options.metabaseUsername, + password: options.metabasePassword, + }); + await runBitbucket({...options, airbyte, metabase}); }); return cmd; @@ -224,6 +230,14 @@ export async function runBitbucket(cfg: BitbucketConfig): Promise { cfg.cutoffDays || DEFAULT_CUTOFF_DAYS, repos?.length || 0 ); + + try { + await cfg.metabase.forceSync(); + } catch (error) { + // main intent is to have filters immediately populated with values + // we do nothing on failure, basic functionalities are not impacted + // daily/hourly metabase db scans will eventually get us there + } } interface Workspace { diff --git a/cli/src/cli.ts b/cli/src/cli.ts index 081d54b9..2c5aaec0 100644 --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -6,11 +6,14 @@ import {makeBitbucketCommand, runBitbucket} from './bitbucket/run'; import {makeGithubCommand, runGithub} from './github/run'; import {makeGitlabCommand, runGitlab} from './gitlab/run'; import {makeJiraCommand, runJira} from './jira/run'; +import {Metabase} from './metabase/metabase-client'; import {display, terminalLink} from './utils'; import {runSelect} from './utils/prompts'; const DEFAULT_AIRBYTE_URL = 'http://localhost:8000'; const DEFAULT_METABASE_URL = 'http://localhost:3000'; +const DEFAULT_METABASE_USER = 'admin@admin.com'; +const DEFAULT_METABASE_PASSWORD = 'admin'; export function wrapApiError(cause: unknown, msg: string): Error { // Omit verbose axios error @@ -33,6 +36,12 @@ export async function main(): Promise { .command('pick-source', {isDefault: true, hidden: true}) .action(async (options) => { const airbyte = new Airbyte(options.airbyteUrl); + const metabase = await Metabase.fromConfig({ + url: options.metabaseUrl, + username: options.metabaseUsername, + password: options.metabasePassword, + }); + let done = false; while (!done) { const source = await runSelect({ @@ -48,16 +57,16 @@ export async function main(): Promise { }); switch (source) { case 'GitHub (Cloud)': - await runGithub({airbyte}); + await runGithub({airbyte, metabase}); break; case 'GitLab (Cloud / Server)': - await runGitlab({airbyte}); + await runGitlab({airbyte, metabase}); break; case 'Bitbucket (Cloud / Server)': - await runBitbucket({airbyte}); + await runBitbucket({airbyte, metabase}); break; case 'Jira (Cloud)': - await runJira({airbyte}); + await runJira({airbyte, metabase}); break; case 'I\'m done!': done = true; @@ -69,6 +78,16 @@ export async function main(): Promise { cmd.option('--airbyte-url ', 'Airbyte URL', DEFAULT_AIRBYTE_URL); cmd .option('--metabase-url ', 'Metabase URL', DEFAULT_METABASE_URL) + .option( + '--metabase-username ', + 'Metabase username', + DEFAULT_METABASE_USER + ) + .option( + '--metabase-password ', + 'Metabase password', + DEFAULT_METABASE_PASSWORD + ) .hook('postAction', async (thisCommand) => { display( `Check out your metrics in ${await terminalLink( diff --git a/cli/src/github/run.ts b/cli/src/github/run.ts index cae61596..5bcc1d57 100644 --- a/cli/src/github/run.ts +++ b/cli/src/github/run.ts @@ -4,6 +4,7 @@ import VError from 'verror'; import {Airbyte} from '../airbyte/airbyte-client'; import {wrapApiError} from '../cli'; +import {Metabase} from '../metabase/metabase-client'; import { display, Emoji, @@ -26,6 +27,7 @@ const DEFAULT_CUTOFF_DAYS = 30; interface GithubConfig { readonly airbyte: Airbyte; + readonly metabase: Metabase; readonly token?: string; readonly repoList?: ReadonlyArray; readonly cutoffDays?: number; @@ -51,8 +53,13 @@ export function makeGithubCommand(): Command { cmd.action(async (options) => { const airbyte = new Airbyte(options.airbyteUrl); + const metabase = await Metabase.fromConfig({ + url: options.metabaseUrl, + username: options.metabaseUsername, + password: options.metabasePassword, + }); - await runGithub({...options, airbyte}); + await runGithub({...options, airbyte, metabase}); }); return cmd; @@ -143,6 +150,14 @@ export async function runGithub(cfg: GithubConfig): Promise { cfg.cutoffDays || DEFAULT_CUTOFF_DAYS, repos?.length || 0 ); + + try { + await cfg.metabase.forceSync(); + } catch (error) { + // main intent is to have filters immediately populated with values + // we do nothing on failure, basic functionalities are not impacted + // daily/hourly metabase db scans will eventually get us there + } } async function promptForRepos(token: string): Promise> { diff --git a/cli/src/gitlab/run.ts b/cli/src/gitlab/run.ts index c25ca6d6..4b3af5db 100644 --- a/cli/src/gitlab/run.ts +++ b/cli/src/gitlab/run.ts @@ -4,6 +4,7 @@ import VError from 'verror'; import {Airbyte} from '../airbyte/airbyte-client'; import {wrapApiError} from '../cli'; +import {Metabase} from '../metabase/metabase-client'; import { display, Emoji, @@ -28,6 +29,7 @@ const DEFAULT_API_URL = 'gitlab.com'; interface GitLabConfig { readonly airbyte: Airbyte; + readonly metabase: Metabase; readonly apiUrl?: string; readonly token?: string; readonly projectList?: ReadonlyArray; @@ -55,8 +57,13 @@ export function makeGitlabCommand(): Command { cmd.action(async (options) => { const airbyte = new Airbyte(options.airbyteUrl); + const metabase = await Metabase.fromConfig({ + url: options.metabaseUrl, + username: options.metabaseUsername, + password: options.metabasePassword, + }); - await runGitlab({...options, airbyte}); + await runGitlab({...options, airbyte, metabase}); }); return cmd; @@ -151,6 +158,14 @@ export async function runGitlab(cfg: GitLabConfig): Promise { cfg.cutoffDays || DEFAULT_CUTOFF_DAYS, projects?.length || 0 ); + + try { + await cfg.metabase.forceSync(); + } catch (error) { + // main intent is to have filters immediately populated with values + // we do nothing on failure, basic functionalities are not impacted + // daily/hourly metabase db scans will eventually get us there + } } async function promptForProjects( diff --git a/cli/src/jira/run.ts b/cli/src/jira/run.ts index 2c206569..bf83770a 100644 --- a/cli/src/jira/run.ts +++ b/cli/src/jira/run.ts @@ -4,6 +4,7 @@ import VError from 'verror'; import {Airbyte} from '../airbyte/airbyte-client'; import {wrapApiError} from '../cli'; +import {Metabase} from '../metabase/metabase-client'; import { display, Emoji, @@ -27,6 +28,7 @@ const DEFAULT_CUTOFF_DAYS = 30; interface JiraConfig { readonly airbyte: Airbyte; + readonly metabase: Metabase; readonly email?: string; readonly token?: string; readonly domain?: string; @@ -55,8 +57,13 @@ export function makeJiraCommand(): Command { cmd.action(async (options) => { const airbyte = new Airbyte(options.airbyteUrl); + const metabase = await Metabase.fromConfig({ + url: options.metabaseUrl, + username: options.metabaseUsername, + password: options.metabasePassword, + }); - await runJira({...options, airbyte}); + await runJira({...options, airbyte, metabase}); }); return cmd; @@ -171,6 +178,14 @@ export async function runJira(cfg: JiraConfig): Promise { cfg.cutoffDays || DEFAULT_CUTOFF_DAYS, projects?.length || 0 ); + + try { + await cfg.metabase.forceSync(); + } catch (error) { + // main intent is to have filters immediately populated with values + // we do nothing on failure, basic functionalities are not impacted + // daily/hourly metabase db scans will eventually get us there + } } async function promptForProjects( diff --git a/cli/src/metabase/metabase-client.ts b/cli/src/metabase/metabase-client.ts new file mode 100644 index 00000000..63a8e918 --- /dev/null +++ b/cli/src/metabase/metabase-client.ts @@ -0,0 +1,56 @@ +import axios, {AxiosInstance} from 'axios'; +import {VError} from 'verror'; + +export function wrapApiError(cause: unknown, msg: string): Error { + // Omit verbose axios error + const truncated = new VError((cause as Error).message); + return new VError(truncated, msg); +} + +export interface MetabaseConfig { + readonly url: string; + readonly username: string; + readonly password: string; +} + +export class Metabase { + constructor(private readonly api: AxiosInstance) {} + + static async fromConfig(cfg: MetabaseConfig): Promise { + const token = await Metabase.sessionToken(cfg); + const api = axios.create({ + baseURL: `${cfg.url}/api`, + headers: { + 'X-Metabase-Session': token, + }, + }); + return new Metabase(api); + } + + private static async sessionToken(cfg: MetabaseConfig): Promise { + const {url, username, password} = cfg; + try { + const {data} = await axios.post(`${url}/api/session`, { + username, + password, + }); + return data.id; + } catch (err) { + throw wrapApiError(err, 'failed to get session token'); + } + } + + async forceSync(): Promise { + try { + // Objective is to get filter values populated + // Faros is always has DB id 2 + // note that API call rescan_value was not sufficient + // hence the call to sync_schema instead + // https://www.metabase.com/docs/latest/api/database + const {data} = await this.api.post('database/2/sync_schema'); + return data; + } catch (err) { + throw wrapApiError(err, 'unable to trigger rescan'); + } + } +} diff --git a/init/resources/metabase/dashboards/git.json b/init/resources/metabase/dashboards/git.json index 89287e2b..9095ead4 100644 --- a/init/resources/metabase/dashboards/git.json +++ b/init/resources/metabase/dashboards/git.json @@ -1600,8 +1600,7 @@ "name": "Repository", "slug": "repository", "id": "81276e50", - "type": "string/=", - "sectionId": "string" + "type": "category" } ], "layout": [ @@ -2202,6 +2201,16 @@ "visualization_settings": {} } ], + "fields": [ + { + "field": {{ field "vcs_Repository.name" }}, + "type": "type/Category" + }, + { + "field": {{ field "tms_TaskBoard.name" }}, + "type": "type/Category" + } + ], "path": "/Faros CE/Git", "priority": 14, "bookmark": true diff --git a/start.sh b/start.sh index a3a3fe21..5eeae420 100755 --- a/start.sh +++ b/start.sh @@ -76,7 +76,12 @@ main() { docker pull farosai/faros-ce-cli:latest AIRBYTE_URL=$(grep "^WEBAPP_URL" .env| sed 's/^WEBAPP_URL=//') METABASE_PORT=$(grep "^METABASE_PORT" .env| sed 's/^METABASE_PORT=//') - docker run --network host -it farosai/faros-ce-cli pick-source --airbyte-url "$AIRBYTE_URL" --metabase-url "http://localhost:$METABASE_PORT" + METABASE_USER=$(grep "^METABASE_USER" .env| sed 's/^METABASE_USER=//') + METABASE_PASSWORD=$(grep "^METABASE_PASSWORD" .env| sed 's/^METABASE_PASSWORD=//') + docker run --network host -it farosai/faros-ce-cli pick-source --airbyte-url "$AIRBYTE_URL" \ + --metabase-url "http://localhost:$METABASE_PORT" \ + --metabase-username "$METABASE_USER" \ + --metabase-password "$METABASE_PASSWORD" fi }