diff --git a/.github/workflows/tinybird_cd.yml b/.github/workflows/tinybird_cd.yml new file mode 100644 index 00000000..6050c1e9 --- /dev/null +++ b/.github/workflows/tinybird_cd.yml @@ -0,0 +1,22 @@ + + ################################################## + ### Visit https://github.com/tinybirdco/ci ### + ### for more details or custom CI/CD ### + ################################################## + + name: Tinybird - CD Workflow + + on: + workflow_dispatch: + push: + branches: + - main + - master + jobs: + cd: # deploy changes to workspace 'joshkennedy_me' + uses: tinybirdco/ci/.github/workflows/cd.yml@v3.1.0 + with: + data_project_dir: . + secrets: + tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN }} # set the Workspace admin token in GitHub secrets + tb_host: https://ui.us-east.aws.tinybird.co diff --git a/.github/workflows/tinybird_ci.yml b/.github/workflows/tinybird_ci.yml new file mode 100644 index 00000000..9d23b877 --- /dev/null +++ b/.github/workflows/tinybird_ci.yml @@ -0,0 +1,27 @@ + + ################################################## + ### Visit https://github.com/tinybirdco/ci ### + ### for more details or custom CI/CD ### + ################################################## + + name: Tinybird - CI Workflow + + on: + workflow_dispatch: + pull_request: + branches: + - main + - master + types: [opened, reopened, labeled, unlabeled, synchronize, closed] + + concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} + + jobs: + ci: # ci using branches from workspace 'joshkennedy_me' + uses: tinybirdco/ci/.github/workflows/ci.yml@v3.1.0 + with: + data_project_dir: . + tb_format: false + secrets: + tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN }} # set the Workspace admin token in GitHub secrets + tb_host: https://ui.us-east.aws.tinybird.co diff --git a/.github/workflows/tinybird_release.yml b/.github/workflows/tinybird_release.yml new file mode 100644 index 00000000..4857e6ef --- /dev/null +++ b/.github/workflows/tinybird_release.yml @@ -0,0 +1,25 @@ + + ################################################## + ### Visit https://github.com/tinybirdco/ci ### + ### for more details or custom CI/CD ### + ################################################## + + name: Tinybird - Releases Workflow + + on: + workflow_dispatch: + inputs: + job_to_run: + description: 'Select the job to run manually' + required: true + default: 'promote' + + jobs: + cd: # manage releases for workspace 'joshkennedy_me' + uses: tinybirdco/ci/.github/workflows/release.yml@v3.1.0 + with: + job_to_run: ${{ inputs.job_to_run }} + data_project_dir: . + secrets: + tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN }} # set the Workspace admin token in GitHub secrets + tb_host: https://ui.us-east.aws.tinybird.co diff --git a/.tinyenv b/.tinyenv new file mode 100644 index 00000000..b58c3697 --- /dev/null +++ b/.tinyenv @@ -0,0 +1,29 @@ + +# VERSION format is major.minor.patch-post where major, minor, patch and post are integer numbers +# bump post to deploy to the current live Release, rollback to previous post version is not available +# bump patch or minor to deploy a new Release and auto-promote it to live. Add TB_AUTO_PROMOTE=0 to create the Release in preview status +# bump major to deploy a new Release in preview status +VERSION=0.0.1 + + + + +########## +# OPTIONAL env vars +# Deploy a new Release in preview status (default is 1) +# TB_AUTO_PROMOTE=0 + +# Force old Releases deletion on promote (default is 0) +# Setting it to 1 will remove oldest rollback Releases even when some resource is still in use +# TB_FORCE_REMOVE_OLDEST_ROLLBACK=0 + +# Don't print CLI version warning message if there's a new available version +# TB_VERSION_WARNING=0 + +# Skip regression tests +# TB_SKIP_REGRESSION=0 + +# Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output. +# OBFUSCATE_REGEX_PATTERN="https://(www\.)?[^/]+||^Follow these instructions =>" +# OBFUSCATE_PATTERN_SEPARATOR=|| +########## diff --git a/datasources/.gitkeep b/datasources/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/datasources/fixtures/.gitkeep b/datasources/fixtures/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deploy/.gitkeep b/deploy/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/endpoints/.gitkeep b/endpoints/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/pipes/.gitkeep b/pipes/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..77a8f531 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +tinybird-cli>=3 \ No newline at end of file diff --git a/scripts/append_fixtures.sh b/scripts/append_fixtures.sh new file mode 100755 index 00000000..8c415b88 --- /dev/null +++ b/scripts/append_fixtures.sh @@ -0,0 +1,23 @@ + +#!/usr/bin/env bash +set -euxo pipefail + +VERSION=$1 + +directory="datasources/fixtures" +extensions=("csv" "ndjson") + +absolute_directory=$(realpath "$directory") + +for extension in "${extensions[@]}"; do + file_list=$(find "$absolute_directory" -type f -name "*.$extension") + + for file_path in $file_list; do + file_name=$(basename "$file_path") + file_name_without_extension="${file_name%.*}" + + command="tb --semver $VERSION datasource append $file_name_without_extension datasources/fixtures/$file_name" + echo $command + $command + done +done diff --git a/scripts/exec_test.sh b/scripts/exec_test.sh new file mode 100755 index 00000000..b509694b --- /dev/null +++ b/scripts/exec_test.sh @@ -0,0 +1,63 @@ + +#!/usr/bin/env bash +set -euxo pipefail + +export TB_VERSION_WARNING=0 +export VERSION=$1 + +run_test() { + t=$1 + echo "** Running $t **" + # Check if VERSION is provided + if [[ -n $VERSION ]]; then + sed -i "s/tb/tb --semver $VERSION/" $t + fi + echo "** $(cat $t)" + tmpfile=$(mktemp) + retries=0 + TOTAL_RETRIES=3 + + # When appending fixtures, we need to retry in case of the data is not replicated in time + while [ $retries -lt $TOTAL_RETRIES ]; do + # Run the test and store the output in a temporary file + bash $t $2 >$tmpfile + exit_code=$? + if [ "$exit_code" -eq 0 ]; then + # If the test passed, break the loop + if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then + break + # If the test failed, increment the retries counter and try again + else + retries=$((retries+1)) + fi + # If the bash command failed, print an error message and break the loop + else + break + fi + done + + if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then + echo "✅ Test $t passed" + rm $tmpfile + return 0 + elif [ $retries -eq $TOTAL_RETRIES ]; then + echo "🚨 ERROR: Test $t failed, diff:"; + diff -B ${t}.result $tmpfile + rm $tmpfile + return 1 + else + echo "🚨 ERROR: Test $t failed with bash command exit code $?" + cat $tmpfile + rm $tmpfile + return 1 + fi + echo "" +} +export -f run_test + +fail=0 +find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} $VERSION || fail=1 + +if [ $fail == 1 ]; then + exit -1; +fi diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 00000000..e69de29b