Sync Data Project

.github/workflows/tinybird_cd.yml

@@ -0,0 +1,22 @@
+
+    ##################################################
+    ###   Visit https://github.com/tinybirdco/ci   ###
+    ###   for more details or custom CI/CD         ###
+    ##################################################
+
+    name: Tinybird - CD Workflow
+
+    on:
+      workflow_dispatch:
+      push:
+        branches:
+          - main
+          - master
+    jobs:
+      cd:  # deploy changes to workspace 'joshkennedy_me'
+        uses: tinybirdco/ci/.github/workflows/[email protected]
+        with:
+          data_project_dir: .
+        secrets:
+          tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN }}  # set the Workspace admin token in GitHub secrets
+          tb_host: https://ui.us-east.aws.tinybird.co

.github/workflows/tinybird_ci.yml

@@ -0,0 +1,27 @@
+
+    ##################################################
+    ###   Visit https://github.com/tinybirdco/ci   ###
+    ###   for more details or custom CI/CD         ###
+    ##################################################
+
+    name: Tinybird - CI Workflow
+
+    on:
+      workflow_dispatch:
+      pull_request:
+        branches:
+          - main
+          - master
+        types: [opened, reopened, labeled, unlabeled, synchronize, closed]
+
+    concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+
+    jobs:
+        ci: # ci using branches from workspace 'joshkennedy_me'
+          uses: tinybirdco/ci/.github/workflows/[email protected]
+          with:
+            data_project_dir: .
+            tb_format: false
+          secrets:
+            tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN }}  # set the Workspace admin token in GitHub secrets
+            tb_host: https://ui.us-east.aws.tinybird.co

.github/workflows/tinybird_release.yml

@@ -0,0 +1,25 @@
+
+    ##################################################
+    ###   Visit https://github.com/tinybirdco/ci   ###
+    ###   for more details or custom CI/CD         ###
+    ##################################################
+
+    name: Tinybird - Releases Workflow
+
+    on:
+      workflow_dispatch:
+        inputs:
+          job_to_run:
+            description: 'Select the job to run manually'
+            required: true
+            default: 'promote'
+
+    jobs:
+      cd:  # manage releases for workspace 'joshkennedy_me'
+        uses: tinybirdco/ci/.github/workflows/[email protected]
+        with:
+          job_to_run: ${{ inputs.job_to_run }}
+          data_project_dir: .
+        secrets:
+          tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN }}  # set the Workspace admin token in GitHub secrets
+          tb_host: https://ui.us-east.aws.tinybird.co

.tinyenv

@@ -0,0 +1,29 @@
+
+# VERSION format is major.minor.patch-post where major, minor, patch and post are integer numbers
+# bump post to deploy to the current live Release, rollback to previous post version is not available
+# bump patch or minor to deploy a new Release and auto-promote it to live. Add TB_AUTO_PROMOTE=0 to create the Release in preview status
+# bump major to deploy a new Release in preview status
+VERSION=0.0.1
+
+
+
+
+##########
+# OPTIONAL env vars
+# Deploy a new Release in preview status (default is 1)
+# TB_AUTO_PROMOTE=0
+
+# Force old Releases deletion on promote (default is 0)
+# Setting it to 1 will remove oldest rollback Releases even when some resource is still in use
+# TB_FORCE_REMOVE_OLDEST_ROLLBACK=0
+
+# Don't print CLI version warning message if there's a new available version
+# TB_VERSION_WARNING=0
+
+# Skip regression tests
+# TB_SKIP_REGRESSION=0
+
+# Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
+# OBFUSCATE_REGEX_PATTERN="https://(www\.)?[^/]+||^Follow these instructions =>"
+# OBFUSCATE_PATTERN_SEPARATOR=||
+##########

requirements.txt

@@ -0,0 +1 @@
+tinybird-cli>=3

scripts/append_fixtures.sh

@@ -0,0 +1,23 @@
+
+#!/usr/bin/env bash
+set -euxo pipefail
+
+VERSION=$1
+
+directory="datasources/fixtures"
+extensions=("csv" "ndjson")
+
+absolute_directory=$(realpath "$directory")
+
+for extension in "${extensions[@]}"; do
+  file_list=$(find "$absolute_directory" -type f -name "*.$extension")
+
+  for file_path in $file_list; do
+    file_name=$(basename "$file_path")
+    file_name_without_extension="${file_name%.*}"
+
+    command="tb --semver $VERSION datasource append $file_name_without_extension datasources/fixtures/$file_name"
+    echo $command
+    $command
+  done
+done

scripts/exec_test.sh

@@ -0,0 +1,63 @@
+
+#!/usr/bin/env bash
+set -euxo pipefail
+
+export TB_VERSION_WARNING=0
+export VERSION=$1
+
+run_test() {
+    t=$1
+    echo "** Running $t **"
+    # Check if VERSION is provided
+    if [[ -n $VERSION ]]; then
+        sed -i "s/tb/tb --semver $VERSION/" $t
+    fi
+    echo "** $(cat $t)"
+    tmpfile=$(mktemp)
+    retries=0
+    TOTAL_RETRIES=3
+
+    # When appending fixtures, we need to retry in case of the data is not replicated in time
+    while [ $retries -lt $TOTAL_RETRIES ]; do
+        # Run the test and store the output in a temporary file
+        bash $t $2 >$tmpfile
+        exit_code=$?
+        if [ "$exit_code" -eq 0 ]; then
+            # If the test passed, break the loop
+            if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+                break
+            # If the test failed, increment the retries counter and try again
+            else
+                retries=$((retries+1))
+            fi
+        # If the bash command failed, print an error message and break the loop
+        else
+            break
+        fi
+    done
+
+    if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+        echo "✅ Test $t passed"
+        rm $tmpfile
+        return 0
+    elif [ $retries -eq $TOTAL_RETRIES ]; then
+        echo "🚨 ERROR: Test $t failed, diff:";
+        diff -B ${t}.result $tmpfile
+        rm $tmpfile
+        return 1
+    else
+        echo "🚨 ERROR: Test $t failed with bash command exit code $?"
+        cat $tmpfile
+        rm $tmpfile
+        return 1
+    fi
+    echo ""
+}
+export -f run_test
+
+fail=0
+find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} $VERSION || fail=1
+
+if [ $fail == 1 ]; then
+  exit -1;
+fi