Skip to content

Commit

Permalink
Merge branch 'airbytehq:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
andyh565 authored Aug 8, 2024
2 parents b0d2e79 + 9d2c99a commit 9834fa4
Show file tree
Hide file tree
Showing 2,588 changed files with 107,863 additions and 60,639 deletions.
2 changes: 1 addition & 1 deletion .bumpversion.cfg
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.63.4
current_version = 0.63.14
commit = False
tag = False
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
Expand Down
21 changes: 18 additions & 3 deletions .github/actions/run-airbyte-ci/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,15 @@ inputs:
description: "Whether the PR is from a fork"
required: false
default: "false"
max_attempts:
description: "Number of attempts at running the airbyte-ci command"
required: false
default: 1
retry_wait_seconds:
description: "Number of seconds to wait between retry attempts"
required: false
default: 60

runs:
using: "composite"
steps:
Expand All @@ -109,7 +118,7 @@ runs:
is_fork: ${{ inputs.is_fork }}
- name: Run airbyte-ci
id: run-airbyte-ci
shell: bash
uses: nick-fields/retry@v3
env:
CI: "True"
CI_GIT_USER: ${{ github.repository_owner }}
Expand Down Expand Up @@ -140,8 +149,14 @@ runs:
SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }}
SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }}
SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }}
run: |
airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
with:
shell: bash
max_attempts: ${{ inputs.max_attempts }}
retry_wait_seconds: ${{ inputs.retry_wait_seconds }}
# 360mn > 6 hours: it's the GitHub runner max job duration
timeout_minutes: 360
command: |
airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
- name: Stop Engine
id: stop-engine
if: always()
Expand Down
125 changes: 125 additions & 0 deletions .github/workflows/approve-regression-tests-command.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
name: Approve Regression Tests
permissions:
pull-requests: write
statuses: write
on:
workflow_dispatch:
inputs:
pr:
description: "Pull request number. Used to pull the proper branch ref, including on forks."
type: number
required: false
comment-id:
description: "Optional. The comment-id of the slash command. Used to update the comment with the status."
required: false
connector-name:
description: "Optional. Name of the connector whose regression tests are approved."
required: false

# These must be declared, but they are unused and ignored.
# TODO: Infer 'repo' and 'gitref' from PR number on other workflows, so we can remove these.
repo:
description: "Repo (Ignored)"
required: false
default: "airbytehq/airbyte"
gitref:
description: "Ref (Ignored)"
required: false

run-name: "Approve Regression Tests #${{ github.event.inputs.pr }}"

jobs:
approve-regression-tests:
name: "Approve Regression Tests"
runs-on: ubuntu-latest
steps:
- name: Get job variables
id: job-vars
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: bash
run: |
PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }})
echo "PR_JSON: $PR_JSON"
echo "repo=$(echo "$PR_JSON" | jq -r .head.repo.full_name)" >> $GITHUB_OUTPUT
BRANCH=$(echo "$PR_JSON" | jq -r .head.ref)
echo "branch=$BRANCH" >> $GITHUB_OUTPUT
echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
LATEST_COMMIT=$(gh api repos/${{ github.repository }}/commits/$BRANCH | jq -r .sha)
echo "latest_commit=$LATEST_COMMIT" >> $GITHUB_OUTPUT
- name: Append comment with job run link
# If comment-id is not provided, this will create a new
# comment with the job run link.
id: first-comment-action
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ github.event.inputs.comment-id }}
issue-number: ${{ github.event.inputs.pr }}
body: |
> [Check job output.][1]
[1]: ${{ steps.job-vars.outputs.run-url }}
- name: Approve regression tests
id: approve-regression-tests
if: github.event.inputs.connector-name != null
run: |
echo "approving regression test status check for ${{ github.event.inputs.connector-name }} if it exists ...."
response=$(curl --write-out '%{http_code}' --silent --output /dev/null \
--request POST \
--url ${{ github.api_url }}/repos/${{ github.repository }}/statuses/${{ steps.job-vars.outputs.latest_commit }} \
--header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
--header 'content-type: application/json' \
--data '{
"state": "success",
"context": "Regression Tests on ${{ github.event.inputs.connector-name }}",
"target_url": "https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/live-tests"
}')
if [ $response -ne 201 ]; then
echo "Failed to approve regression tests for ${{ github.event.inputs.connector-name }}. HTTP status code: $response"
exit 1
else
echo "Regression tests for ${{ github.event.inputs.connector-name }} approved."
fi
- name: Update global regression test approval
id: update-global-regression-test-approval
if: github.event.inputs.connector-name == null
run: |
echo "updating regression test approval status check if it exists ...."
response=$(curl --write-out '%{http_code}' --silent --output /dev/null \
--request POST \
--url ${{ github.api_url }}/repos/${{ github.repository }}/statuses/${{ steps.job-vars.outputs.latest_commit }} \
--header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
--header 'content-type: application/json' \
--data '{
"state": "success",
"context": "Regression Test Results Reviewed and Approved",
"target_url": "https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/live-tests"
}')
if [ $response -ne 201 ]; then
echo "Failed to update regression test approval status check. HTTP status code: $response"
exit 1
else
echo "Regression test status check updated."
fi
- name: Append success comment
uses: peter-evans/create-or-update-comment@v4
if: success()
with:
comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
reactions: "+1"
body: |
> ✅ Approving regression tests
- name: Append failure comment
uses: peter-evans/create-or-update-comment@v4
if: failure()
with:
comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
reactions: confused
body: |
> ❌ Regression test approval failed
1 change: 0 additions & 1 deletion .github/workflows/community_ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ jobs:
filters: |
protected_paths:
- '.github/**'
- 'airbyte-ci/**'
- name: Fail if changes in protected paths
if: steps.check_for_changes_in_protected_paths.outputs.protected_paths == 'true'
Expand Down
23 changes: 18 additions & 5 deletions .github/workflows/connectors_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,13 @@ jobs:
# Note: expressions within a filter are OR'ed
filters: |
connectors:
- '*'
- 'docs/integrations/**/*'
- 'airbyte-ci/**/*'
- 'airbyte-integrations/connectors/**/*'
- 'airbyte-cdk/java/**/*'
- 'buildSrc/**/*'
# The Connector CI Tests is a status check emitted by airbyte-ci
# We make it pass once we have determined that there are no changes to the connectors
# The Connector CI Tests & Regression Test Results Reviewed and Approved are status checks emitted by airbyte-ci
# We make them pass once we have determined that there are no changes to the connectors
- name: "Skip Connectors CI tests"
if: steps.changes.outputs.connectors != 'true' && github.event_name == 'pull_request'
run: |
Expand All @@ -57,6 +57,19 @@ jobs:
"context": "Connectors CI tests",
"target_url": "${{ github.event.workflow_run.html_url }}"
}' \
- name: "Skip Regression Test Results Reviewed and Approved"
if: steps.changes.outputs.connectors != 'true' && github.event_name == 'pull_request'
run: |
curl --request POST \
--url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
--header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
--header 'content-type: application/json' \
--data '{
"state": "success",
"description": "[Skipped]",
"context": "Regression Test Results Reviewed and Approved",
"target_url": "${{ github.event.workflow_run.html_url }}"
}' \
connectors_ci:
needs: changes
Expand All @@ -73,8 +86,7 @@ jobs:
- name: Check PAT rate limits
run: |
./tools/bin/find_non_rate_limited_PAT \
${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
${{ secrets.GH_PAT_MAINTENANCE_OSS }}
- name: Extract branch name [WORKFLOW DISPATCH]
shell: bash
if: github.event_name == 'workflow_dispatch'
Expand Down Expand Up @@ -113,6 +125,7 @@ jobs:
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcp_integration_tester_credentials: ${{ secrets.GCLOUD_INTEGRATION_TESTER }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
git_branch: ${{ github.head_ref }}
git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/connectors_up_to_date.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:
inputs:
connectors-options:
description: "Options to pass to the 'airbyte-ci connectors' command group."
default: "--concurrency=10 --language=python --language=low-code"
default: "--concurrency=10 --language=python --language=low-code --language=manifest-only"
auto-merge:
description: "Whether to auto-merge the PRs created by the action."
default: "false"
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/gradle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ jobs:
- '**/*.gradle'
- '**/*.kt'
- 'airbyte-cdk/java/**/*'
- 'airbyte-cdk/bulk/**/*'
run-check:
needs:
Expand Down
14 changes: 13 additions & 1 deletion .github/workflows/live_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@ on:
description: Use the local CDK when building the target connector
default: "false"
type: boolean
connection_subset:
description: The subset of connections to select from.
required: true
type: choice
options:
- sandboxes
- all

jobs:
live_tests:
Expand Down Expand Up @@ -86,6 +93,11 @@ jobs:
echo "USE_LOCAL_CDK_FLAG=" >> $GITHUB_ENV
fi
- name: Setup Connection Subset Option
if: github.event_name == 'workflow_dispatch'
run: |
echo "CONNECTION_SUBSET=--connector_live_tests.connection-subset=${{ github.event.inputs.connection_subset }}" >> $GITHUB_ENV
- name: Run Live Tests [WORKFLOW DISPATCH]
if: github.event_name == 'workflow_dispatch' # TODO: consider using the matrix strategy (https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs). See https://github.com/airbytehq/airbyte/pull/37659#discussion_r1583380234 for details.
uses: ./.github/actions/run-airbyte-ci
Expand All @@ -102,4 +114,4 @@ jobs:
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=all --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }}
subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=all --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }}
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
with:
subcommand: "metadata deploy orchestrator"
context: "master"
dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
# dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
github_token: ${{ secrets.GITHUB_TOKEN }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
Expand All @@ -40,7 +40,7 @@ jobs:
with:
subcommand: "metadata deploy orchestrator"
context: "manual"
dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
# dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
github_token: ${{ secrets.GITHUB_TOKEN }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
Expand Down
Loading

0 comments on commit 9834fa4

Please sign in to comment.