add twenty-tinybird package (#8030)

twentyhq/private-issues#75 **TLDR** Add twenty-server package, wich contains the last tinybird datasources and pipes used in analytics. This new version of the API endpoints, pipes and datasources are inspired by the implementation of dub.co (https://github.com/dubinc/dub/blob/main/packages/tinybird/). It contains the webhooks analytics, serverless functions duration and serverless functions error count analytics. As well as **In order to test** - Follow the instructions in the README.md on twenty-tinybird using your admin token from twenty_analytics_playground if you want to modify them. - For a better experience add the extension Tinybird support for VsCode - If you want more info about datasources and pipes please read the tinybird docs. --------- Co-authored-by: Félix Malfait <[email protected]> Co-authored-by: Félix Malfait <[email protected]>
twentyhq · Oct 29, 2024 · 8ef4efa · 8ef4efa
1 parent 6fba3e0
commit 8ef4efa
Show file tree

Hide file tree

Showing 22 changed files with 791 additions and 0 deletions.
diff --git a/.github/workflows/ci-tinybird.yaml b/.github/workflows/ci-tinybird.yaml
@@ -0,0 +1,22 @@
+name: CI Tinybird
+on:
+  # Temporarily disabled
+  # push:
+  #   branches:
+  #     - main
+
+  # pull_request:
+
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  ci:
+    uses: tinybirdco/ci/.github/workflows/ci.yml@main
+    with:
+        data_project_dir: packages/twenty-tinybird
+    secrets:
+        tb_admin_token: ${{ secrets.TINYBIRD_ADMIN_TOKEN }} 
+        tb_host: https://api.eu-central-1.aws.tinybird.co
diff --git a/.gitignore b/.gitignore
@@ -29,3 +29,4 @@ storybook-static
 .nyc_output
 test-results/
 dump.rdb
+.tinyb
diff --git a/packages/twenty-tinybird/.gitignore b/packages/twenty-tinybird/.gitignore
@@ -0,0 +1,2 @@
+.tinyb
+.venv
diff --git a/packages/twenty-tinybird/.tinyenv b/packages/twenty-tinybird/.tinyenv
@@ -0,0 +1,18 @@
+
+VERSION=0.0.0
+
+
+
+##########
+# OPTIONAL env vars
+
+# Don't print CLI version warning message if there's a new available version
+# TB_VERSION_WARNING=0
+
+# Skip regression tests
+# TB_SKIP_REGRESSION=0
+
+# Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
+# OBFUSCATE_REGEX_PATTERN="https://(www\.)?[^/]+||^Follow these instructions =>"
+# OBFUSCATE_PATTERN_SEPARATOR=||
+##########
diff --git a/packages/twenty-tinybird/README.md b/packages/twenty-tinybird/README.md
@@ -0,0 +1,42 @@
+
+## How to use
+Create a virtual enviroment and install tinybird
+```sh
+python3 -m venv .venv
+source .venv/bin/activate
+pip install tinybird-cli
+```
+Authenticate using your admin token from your workspace (twenty_analytics_cli_playground)
+```sh
+tb auth -i
+
+** List of available regions:
+[1] us-east4 (gcp) (https://app.us-east.tinybird.co)
+[2] europe-west3 (gcp) (https://app.tinybird.co/gcp/europe-west3)
+[3] us-east-1 (aws) (https://app.tinybird.co/aws/us-east-1)
+[4] us-west-2 (aws) (https://app.tinybird.co/aws/us-west-2)
+[5] eu-central-1 (aws) (https://app.tinybird.co/aws/eu-central-1) <- this
+[0] Cancel
+
+Use region [5]:
+Copy the "admin your@email" token from from https://app.tinybird.co/tokens and paste it here: <pasted Token>
+** Auth successful!
+** Configuration written to .tinyb file, consider adding it to .gitignore
+```
+To sync your changes to Tinybird use:
+```sh
+tb push --force --push-deps
+```
+To pull data from Tinybird use:
+```sh
+tb pull
+```
+Things I learned:
+
+* When creating Materialied Views think about populating it first using the files from the fixtures
+* When pushing your pipes prefer to push one by one so you can have more coherent error messages
+* The Include files will be nodes that will be added at the start of your node in Tinybird UI
+* It's best to stick only with only the CLI or the UI when developping in Tinybird
+* If you want to format the data us the tb fmt command, perhaps we can add it to the linter in a next pull request.
+
+
diff --git a/packages/twenty-tinybird/datasources/event.datasource b/packages/twenty-tinybird/datasources/event.datasource
@@ -0,0 +1,11 @@
+SCHEMA >
+    `action` String `json:$.action`,
+    `timestamp` DateTime64(3) `json:$.timestamp`,
+    `version` String `json:$.version`,
+    `userId` String `json:$.userId` DEFAULT '',
+    `workspaceId` String `json:$.workspaceId` DEFAULT '',
+    `payload` String `json:$.payload`
+
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYear(timestamp)
+ENGINE_SORTING_KEY action, timestamp
diff --git a/packages/twenty-tinybird/datasources/fixtures/event.csv b/packages/twenty-tinybird/datasources/fixtures/event.csv
diff --git a/packages/twenty-tinybird/datasources/fixtures/pageview.csv b/packages/twenty-tinybird/datasources/fixtures/pageview.csv
@@ -0,0 +1,11 @@
+"href","locale","pathname","referrer","sessionId","timeZone","timestamp","userAgent","userId","version","workspaceId"
+"http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","en-US","/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","ebb6279c-c9a6-44c7-b751-6ee615e6a5fe","Europe/Paris","2024-10-17 12:30:33.604","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36","20202020-9e3b-46d4-a556-88b9ddc2b034","1","20202020-1c25-4d02-bf25-6aeccf7ea419"
+"http://localhost:3001/objects/companies","en-US","/welcome","http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","8fc30143-b648-4fc0-afe7-e55e1c452003","Europe/Paris","2024-10-17 12:37:53.093","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36","20202020-9e3b-46d4-a556-88b9ddc2b034","1","20202020-1c25-4d02-bf25-6aeccf7ea419"
+"http://localhost:3001/objects/companies","en-US","/objects/companies","http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","8fc30143-b648-4fc0-afe7-e55e1c452003","Europe/Paris","2024-10-17 12:37:53.184","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36","20202020-9e3b-46d4-a556-88b9ddc2b034","1","20202020-1c25-4d02-bf25-6aeccf7ea419"
+"http://localhost:3001/settings/profile","en-US","/settings/profile","http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","8fc30143-b648-4fc0-afe7-e55e1c452003","Europe/Paris","2024-10-17 12:37:54.802","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36","20202020-9e3b-46d4-a556-88b9ddc2b034","1","20202020-1c25-4d02-bf25-6aeccf7ea419"
+"http://localhost:3001/settings/integrations","en-US","/settings/integrations","http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","8fc30143-b648-4fc0-afe7-e55e1c452003","Europe/Paris","2024-10-17 12:37:56.686","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36","20202020-9e3b-46d4-a556-88b9ddc2b034","1","20202020-1c25-4d02-bf25-6aeccf7ea419"
+"http://localhost:3001/settings/developers","en-US","/settings/developers","http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","8fc30143-b648-4fc0-afe7-e55e1c452003","Europe/Paris","2024-10-17 12:38:01.364","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36","20202020-9e3b-46d4-a556-88b9ddc2b034","1","20202020-1c25-4d02-bf25-6aeccf7ea419"
+"http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","en-US","/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","8fc30143-b648-4fc0-afe7-e55e1c452003","Europe/Paris","2024-10-17 12:38:02.518","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36","20202020-9e3b-46d4-a556-88b9ddc2b034","1","20202020-1c25-4d02-bf25-6aeccf7ea419"
+"http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","en-US","/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","8fc30143-b648-4fc0-afe7-e55e1c452003","Europe/Paris","2024-10-17 12:41:11.844","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36","20202020-9e3b-46d4-a556-88b9ddc2b034","1","20202020-1c25-4d02-bf25-6aeccf7ea419"
+"http://localhost:3001/settings/profile","en-US","/settings/profile","http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","8fc30143-b648-4fc0-afe7-e55e1c452003","Europe/Paris","2024-10-17 12:41:23.864","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36","20202020-9e3b-46d4-a556-88b9ddc2b034","1","20202020-1c25-4d02-bf25-6aeccf7ea419"
+"http://localhost:3001/settings/appearance","en-US","/settings/appearance","http://localhost:3001/settings/developers/webhooks/41a8ad80-265a-425a-93da-35452d0ac83d","8fc30143-b648-4fc0-afe7-e55e1c452003","Europe/Paris","2024-10-17 12:41:25.972","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36","20202020-9e3b-46d4-a556-88b9ddc2b034","1","20202020-1c25-4d02-bf25-6aeccf7ea419"
diff --git a/packages/twenty-tinybird/datasources/pageview.datasource b/packages/twenty-tinybird/datasources/pageview.datasource
@@ -0,0 +1,16 @@
+SCHEMA >
+    `href` String `json:$.href`,
+    `locale` String `json:$.locale`,
+    `pathname` String `json:$.pathname`,
+    `referrer` String `json:$.referrer`,
+    `sessionId` String `json:$.sessionId`,
+    `timeZone` String `json:$.timeZone`,
+    `timestamp` DateTime64(3) `json:$.timestamp`,
+    `userAgent` String `json:$.userAgent`,
+    `userId` String `json:$.userId` DEFAULT '',
+    `version` String `json:$.version`,
+    `workspaceId` String `json:$.workspaceId`
+
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYear(timestamp)
+ENGINE_SORTING_KEY timestamp, userId, version, workspaceId
diff --git a/packages/twenty-tinybird/datasources/serverlessFunctionEventMV.datasource b/packages/twenty-tinybird/datasources/serverlessFunctionEventMV.datasource
@@ -0,0 +1,11 @@
+SCHEMA >
+    `timestamp` DateTime64(3),
+    `workspaceId` String,
+    `functionId` String,
+    `durationInMs` Int64,
+    `success` Bool,
+    `errorType` String
+
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYYYYMM(timestamp)
+ENGINE_SORTING_KEY timestamp, functionId, success
diff --git a/packages/twenty-tinybird/datasources/webhookEventMV.datasource b/packages/twenty-tinybird/datasources/webhookEventMV.datasource
@@ -0,0 +1,13 @@
+SCHEMA >
+    `timestamp` DateTime64(3),
+    `workspaceId` String,
+    `webhookId` String,
+    `url` String,
+    `success` UInt8,
+    `status` Int64,
+    `eventName` String,
+    `version` String
+
+ENGINE MergeTree
+ENGINE_PARTITION_KEY toYYYYMM(timestamp)
+ENGINE_SORTING_KEY timestamp, workspaceId
diff --git a/packages/twenty-tinybird/includes/timeSeries.incl b/packages/twenty-tinybird/includes/timeSeries.incl
@@ -0,0 +1,87 @@
+DESCRIPTION >
+    Inspired by DUB implementation
+
+NODE dayIntervals
+SQL >
+    %
+    WITH
+        toStartOfDay(
+            toDateTime64({{ DateTime64(start, '2024-10-16 00:00:00.000') }}, 3),
+            {{ String(timezone, 'UTC') }}
+        ) AS start,
+        toStartOfDay(
+            toDateTime64({{ DateTime64(end, '2024-10-23 00:00:00.000') }}, 3),
+            {{ String(timezone, 'UTC') }}
+        ) AS
+    end
+    SELECT
+        arrayJoin(
+            arrayMap(
+                x -> toDateTime64(toStartOfDay(toDateTime64(x, 3), {{ String(timezone, 'UTC') }}), 3),
+                range(toUInt32(start + 86400), toUInt32(end + 86400),
+                86400
+            )
+        )
+    ) as interval
+
+NODE hourIntervals
+SQL >
+    %
+    WITH
+        toStartOfHour(
+            toDateTime64({{ DateTime64(start, '2024-10-22 00:00:00.000') }}, 3),
+            {{ String(timezone, 'UTC') }}
+        ) AS start,
+        toStartOfHour(
+            toDateTime64({{ DateTime64(end, '2024-10-23 00:00:00.000') }}, 3),
+            {{ String(timezone, 'UTC') }}
+        ) AS
+    end
+    SELECT
+        arrayJoin(
+            arrayMap(x -> toDateTime64(x, 3), range(toUInt32(start + 3600), toUInt32(end + 3600), 3600)
+        )
+    ) as interval
+
+NODE customIntervals
+SQL >
+    %
+    WITH
+        time_series AS (
+            SELECT
+                toDateTime64(
+                    toDateTime(
+                        toStartOfInterval(
+                            toDateTime64({{ DateTime64(start, '2024-10-22 00:00:00.000') }}, 3),
+                            INTERVAL {{ Int32(tickIntervalInMinutes, 420) }} MINUTE
+                        )
+                    )
+                    + INTERVAL number * {{ Int32(tickIntervalInMinutes, 420) }} MINUTE,
+                    3
+                ) AS interval
+            FROM
+                numbers(
+                    0,
+                    1 + intDiv(
+                        dateDiff(
+                            'minute',
+                            toDateTime64({{ DateTime64(start, '2024-10-22 00:00:00.000') }}, 3),
+                            toDateTime64({{ DateTime64(end, '2024-10-23 00:00:00.000') }}, 3)
+                        ),
+                        {{ Int32(tickIntervalInMinutes, 420) }}
+                    )
+                )
+            WHERE interval <= toDateTime64({{ DateTime64(end, '2024-10-23 00:00:00.000') }}, 3)
+        )
+    SELECT interval
+    FROM time_series
+
+NODE selectIntervalByGranularity
+SQL >
+    %
+    SELECT *
+    FROM
+        {% if granularity == "custom" %} customIntervals
+        {% elif granularity == "hour" %} hourIntervals
+        {% else %} dayIntervals
+        {% end %}
diff --git a/packages/twenty-tinybird/pipes/getServerlessFunctionDuration.pipe b/packages/twenty-tinybird/pipes/getServerlessFunctionDuration.pipe
@@ -0,0 +1,35 @@
+INCLUDE "../includes/timeSeries.incl"
+
+NODE timeSeriesServerlessFunctionDurationData
+SQL >
+    %
+    SELECT
+        {% if granularity == "hour" %} toStartOfHour(timestamp, {{ String(timezone, 'UTC') }})
+        {% elif granularity == "custom" %}
+            toDateTime64(
+                toStartOfMinute(timestamp, {{ String(timezone, 'UTC') }}),
+                3,
+                {{ String(timezone, 'UTC') }}
+            )
+        {% else %} toDateTime64(toStartOfDay(timestamp, {{ String(timezone, 'UTC') }}), 3)
+        {% end %} AS interval,
+        avg(CAST(durationInMs AS Float64)) as average,
+        min(durationInMs) as minimum,
+        max(durationInMs) as maximum
+    FROM serverlessFunctionEventMV
+    WHERE
+        true
+        AND functionId = {{ String(functionId, 'a9fd87c0-af86-4e17-be3a-a6d3d961678a', required=True) }}
+        AND workspaceId
+        ={{ String(workspaceId, '20202020-1c25-4d02-bf25-6aeccf7ea419', required=True) }}
+        AND timestamp >= {{ DateTime(start, '2024-10-22 00:00:00') }}
+        AND timestamp < {{ DateTime(end, '2024-10-23 00:00:00') }}
+    GROUP BY interval
+    ORDER BY interval
+
+NODE endpoint
+SQL >
+    %
+    SELECT formatDateTime(interval, '%FT%T.000%z') as start, minimum, maximum, average
+    FROM selectIntervalByGranularity
+    LEFT JOIN timeSeriesServerlessFunctionDurationData USING interval
diff --git a/packages/twenty-tinybird/pipes/getServerlessFunctionErrors.pipe b/packages/twenty-tinybird/pipes/getServerlessFunctionErrors.pipe
@@ -0,0 +1,41 @@
+INCLUDE "../includes/timeSeries.incl"
+
+NODE timeSeriesServerlessFunctionErrorsData
+SQL >
+    %
+    SELECT
+        {% if granularity == "hour" %} toStartOfHour(timestamp, {{ String(timezone, 'UTC') }})
+        {% elif granularity == "custom" %}
+            toDateTime64(
+                toStartOfMinute(timestamp, {{ String(timezone, 'UTC') }}),
+                3,
+                {{ String(timezone, 'UTC') }}
+            )
+        {% else %} toDateTime64(toStartOfDay(timestamp, {{ String(timezone, 'UTC') }}), 3)
+        {% end %} AS interval,
+        uniqIf(*, success = false) as error_count,
+        round(
+            if(
+                uniqIf(*, success = true) = 0,
+                0,
+                (uniqIf(*, success = true) - uniqIf(*, success = false)) / uniqIf(*, success = true)
+            ),
+            2
+        ) as success_rate
+    FROM serverlessFunctionEventMV
+    WHERE
+        true
+        AND functionId = {{ String(functionId, 'a9fd87c0-af86-4e17-be3a-a6d3d961678a', required=True) }}
+        AND workspaceId
+        ={{ String(workspaceId, '20202020-1c25-4d02-bf25-6aeccf7ea419', required=True) }}
+        AND timestamp >= {{ DateTime(start, '2024-10-22 00:00:00') }}
+        AND timestamp < {{ DateTime(end, '2024-10-23 00:00:00') }}
+    GROUP BY interval
+    ORDER BY interval
+
+NODE endpoint
+SQL >
+    %
+    SELECT formatDateTime(interval, '%FT%T.000%z') as start, error_count, success_rate
+    FROM selectIntervalByGranularity
+    LEFT JOIN timeSeriesServerlessFunctionErrorsData USING interval
diff --git a/packages/twenty-tinybird/pipes/getWebhookAnalytics.pipe b/packages/twenty-tinybird/pipes/getWebhookAnalytics.pipe
@@ -0,0 +1,34 @@
+INCLUDE "../includes/timeSeries.incl"
+
+NODE timeSeriesWebhookData
+SQL >
+    %
+    SELECT
+        {% if granularity == "hour" %} toStartOfHour(timestamp, {{ String(timezone, 'UTC') }})
+        {% elif granularity == "custom" %}
+            toDateTime64(
+                toStartOfMinute(timestamp, {{ String(timezone, 'UTC') }}),
+                3,
+                {{ String(timezone, 'UTC') }}
+            )
+        {% else %} toDateTime64(toStartOfDay(timestamp, {{ String(timezone, 'UTC') }}), 3)
+        {% end %} AS interval,
+        uniqIf(*, success = true) as success_count,
+        uniqIf(*, success = false) as failure_count
+    FROM webhookEventMV
+    WHERE
+        true
+        AND webhookId = {{ String(webhookId, '90f12aed-0276-4bea-bcaa-c21ea2763d7d', required=True) }}
+        AND workspaceId
+        ={{ String(workspaceId, '20202020-1c25-4d02-bf25-6aeccf7ea419', required=True) }}
+        AND timestamp >= {{ DateTime(start, '2024-10-22 00:00:00') }}
+        AND timestamp < {{ DateTime(end, '2024-10-23 00:00:00') }}
+    GROUP BY interval
+    ORDER BY interval
+
+NODE endpoint
+SQL >
+    %
+    SELECT formatDateTime(interval, '%FT%T.000%z') as start, success_count, failure_count
+    FROM selectIntervalByGranularity
+    LEFT JOIN timeSeriesWebhookData USING interval
diff --git a/packages/twenty-tinybird/pipes/materializeServerlessFunctionEvent.pipe b/packages/twenty-tinybird/pipes/materializeServerlessFunctionEvent.pipe
@@ -0,0 +1,14 @@
+NODE mv
+SQL >
+    SELECT
+        timestamp,
+        workspaceId,
+        JSONExtractString(payload, 'functionId') as functionId,
+        JSONExtractInt(payload, 'duration') as durationInMs,
+        if(JSONExtractString(payload, 'status') = 'SUCCESS', TRUE, FALSE) as success,
+        JSONExtractString(payload, 'errorType') as errorType
+    FROM event
+    WHERE action = 'serverlessFunction.executed'
+
+TYPE MATERIALIZED
+DATASOURCE serverlessFunctionEventMV
diff --git a/packages/twenty-tinybird/pipes/materializeWebhookEvent.pipe b/packages/twenty-tinybird/pipes/materializeWebhookEvent.pipe
@@ -0,0 +1,16 @@
+NODE mv
+SQL >
+    SELECT
+        timestamp,
+        workspaceId,
+        JSONExtractString(payload, 'webhookId') as webhookId,
+        JSONExtractString(payload, 'url') as url,
+        JSONExtractBool(payload, 'success') as success,
+        JSONExtractInt(payload, 'status') as status,
+        JSONExtractString(payload, 'eventName') as eventName,
+        version
+    FROM event
+    WHERE action = 'webhook.response'
+
+TYPE MATERIALIZED
+DATASOURCE webhookEventMV
diff --git a/packages/twenty-tinybird/requirements.txt b/packages/twenty-tinybird/requirements.txt
@@ -0,0 +1 @@
+tinybird-cli==5.10.1