Checkpointing: Worker use destination (instead of source) for state (… #7
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Airbyte CI | |
| on: | |
| schedule: | |
| - cron: '0 */6 * * *' | |
| push: | |
| jobs: | |
| launch_integration_tests: | |
| runs-on: ubuntu-latest | |
| if: github.ref == 'refs/heads/master' | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v2 | |
| - name: Launch Integration Tests | |
| run: ./tools/bin/ci_integration_workflow_launcher.sh | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.SLASH_COMMAND_PAT }} | |
| ## Gradle Build | |
| # In case of self-hosted EC2 errors, remove the `start-build-runner` block. | |
| start-build-runner: | |
| name: Start Build EC2 Runner | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| - name: Start EC2 Runner | |
| id: start-ec2-runner | |
| uses: machulav/[email protected] | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} | |
| ec2-image-id: ami-04bd6e81239f4f3fb | |
| ec2-instance-type: c5.2xlarge | |
| subnet-id: subnet-0469a9e68a379c1d3 | |
| security-group-id: sg-0793f3c9413f21970 | |
| build: | |
| # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. | |
| needs: start-build-runner # required to start the main job when the runner is ready | |
| runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner | |
| # runs-on: ubuntu-latest | |
| name: Build Airbyte | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v2 | |
| - name: Check images exist | |
| run: ./tools/bin/check_images_exist.sh | |
| - name: Pip Caching | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| key: ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}- | |
| - name: Npm Caching | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.npm | |
| key: ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}- | |
| # this intentionally does not use restore-keys so we don't mess with gradle caching | |
| - name: Gradle and Python Caching | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.gradle/caches | |
| ~/.gradle/wrapper | |
| **/.venv | |
| key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: '14' | |
| - uses: actions/setup-node@v1 | |
| with: | |
| node-version: '14.7' | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.7' | |
| - name: Generate Template scaffold | |
| run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan | |
| - name: Format | |
| run: ./gradlew --no-daemon format --scan | |
| - name: Ensure no file change | |
| run: git status --porcelain && test -z "$(git status --porcelain)" | |
| - name: Build | |
| run: CORE_ONLY=true ./gradlew --no-daemon build --scan | |
| - name: Ensure no file change | |
| run: git status --porcelain && test -z "$(git status --porcelain)" | |
| - name: Check documentation | |
| if: success() && github.ref == 'refs/heads/master' | |
| run: ./tools/site/link_checker.sh check_docs | |
| - name: Get Docker Space | |
| run: docker run --rm busybox df -h | |
| - name: Build Core Docker Images | |
| if: success() && github.ref == 'refs/heads/master' | |
| run: ./gradlew --no-daemon composeBuild --scan | |
| env: | |
| GIT_REVISION: ${{ github.sha }} | |
| - name: Image Cleanup | |
| run: ./tools/bin/clean_images.sh | |
| # make sure these always run before pushing core docker images | |
| - name: Run End-to-End Acceptance Tests | |
| if: success() && github.ref == 'refs/heads/master' | |
| run: ./tools/bin/acceptance_test.sh | |
| - name: Push Core Docker Images | |
| if: success() && github.ref == 'refs/heads/master' | |
| run: | | |
| docker login -u airbytebot -p ${DOCKER_PASSWORD} | |
| VERSION=dev docker-compose -f docker-compose.build.yaml push | |
| env: | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Slack Notification - Failure | |
| if: failure() && github.ref == 'refs/heads/master' | |
| uses: rtCamp/action-slack-notify@master | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} | |
| SLACK_USERNAME: Buildozer | |
| SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg | |
| SLACK_COLOR: DC143C | |
| SLACK_TITLE: "Build failure" | |
| SLACK_FOOTER: "" | |
| - name: Slack Notification - Success | |
| if: success() && github.ref == 'refs/heads/master' | |
| uses: rtCamp/action-slack-notify@master | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} | |
| SLACK_USERNAME: Buildbot | |
| SLACK_TITLE: "Build Success" | |
| SLACK_FOOTER: "" | |
| # In case of self-hosted EC2 errors, remove the `stop-build-runner` block. | |
| stop-build-runner: | |
| name: Stop Build EC2 Runner | |
| needs: | |
| - start-build-runner # required to get output from the start-runner job | |
| - build # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| - name: Stop EC2 runner | |
| uses: machulav/[email protected] | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} | |
| label: ${{ needs.start-build-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} | |
| ## Acceptance Test | |
| # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. | |
| start-acceptance-test-runner: | |
| name: Start Acceptance Test EC2 Runner | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| - name: Start EC2 runner | |
| id: start-ec2-runner | |
| uses: machulav/[email protected] | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} | |
| ec2-image-id: ami-04bd6e81239f4f3fb | |
| ec2-instance-type: c5.2xlarge | |
| subnet-id: subnet-0469a9e68a379c1d3 | |
| security-group-id: sg-0793f3c9413f21970 | |
| acceptance-test: | |
| # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. | |
| needs: start-acceptance-test-runner # required to start the main job when the runner is ready | |
| runs-on: ${{ needs.start-acceptance-test-runner.outputs.label }} # run the job on the newly created runner | |
| # runs-on: ubuntu-latest | |
| name: Run Acceptance Tests | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: '14' | |
| - uses: actions/setup-node@v1 | |
| with: | |
| node-version: '14.7' | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.7' | |
| - name: Build Core Docker Images and Run Tests | |
| run: CORE_ONLY=true ./gradlew --no-daemon composeBuild test -x :airbyte-webapp:test --scan | |
| env: | |
| GIT_REVISION: ${{ github.sha }} | |
| CORE_ONLY: true | |
| - name: Run Docker End-to-End Acceptance Tests | |
| run: | | |
| ./tools/bin/acceptance_test.sh | |
| # In case of self-hosted EC2 errors, remove the `stop-build-runner` block. | |
| stop-acceptance-test-runner: | |
| name: Stop Acceptance Test EC2 Runner | |
| needs: | |
| - start-acceptance-test-runner # required to get output from the start-runner job | |
| - acceptance-test # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| - name: Stop EC2 runner | |
| uses: machulav/[email protected] | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} | |
| label: ${{ needs.start-acceptance-test-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-acceptance-test-runner.outputs.ec2-instance-id }} | |
| ## Frontend Test | |
| frontend-test: | |
| runs-on: ubuntu-latest | |
| name: Run Frontend Test | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v2 | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: '14' | |
| - uses: actions/setup-node@v1 | |
| with: | |
| node-version: '14.7' | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.7' | |
| - name: Build Core Docker Images and Run Tests | |
| run: CORE_ONLY=true ./gradlew --no-daemon build --scan | |
| - name: Run End-to-End Frontend Tests | |
| run: ./tools/bin/e2e_test.sh | |
| # DISABLED UNTIL WE HAVE TEMPORAL ON KUBE | |
| # test_kube: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - name: Checkout Airbyte | |
| # uses: actions/checkout@v2 | |
| # | |
| # - uses: actions/setup-java@v1 | |
| # with: | |
| # java-version: '14' | |
| # | |
| # - uses: actions/setup-node@v1 | |
| # with: | |
| # node-version: '14.7' | |
| # | |
| # - uses: actions/setup-python@v2 | |
| # with: | |
| # python-version: '3.7' | |
| # | |
| # - name: Setup Minikube | |
| # uses: manusa/[email protected] | |
| # with: | |
| # minikube version: 'v1.16.0' | |
| # kubernetes version: 'v1.19.2' | |
| # | |
| # - name: Install socat | |
| # run: sudo apt-get install socat | |
| # | |
| # - name: Build Core Docker Images and Run Tests | |
| # run: CORE_ONLY=true ./gradlew --no-daemon composeBuild test -x :airbyte-webapp:test --scan | |
| # env: | |
| # GIT_REVISION: ${{ github.sha }} | |
| # CORE_ONLY: true | |
| # | |
| # - name: Run Kubernetes End-to-End Acceptance Tests | |
| # run: | | |
| # ./tools/bin/acceptance_test_kube.sh |