Skip to content

Commit

Permalink
Migrate CircleCI to GitHub Actions (#162)
Browse files Browse the repository at this point in the history
* Migrate CircleCI to GitHub Actions

* Add support for unit tests
* Update dependencies to match `dbt-snowflake`
* Update tox config to match `dbt-snowflake`
* Update setup.py to match `dbt-snowflake`

* Fix the "build packages" CI job

* Unlike other DBT adapters, this one contains support for three
  different flavors of MySQL, thus has a handful of duplicate files.
  Therefore W002 should be ignored, as duplicates are not an error.

* Add GitHub actions integration tests

* Used `dbt-snowflake` as an example

* Add codeowners file

* Add tests badges

* Use markdown badges instead

* Temporarily run integration tests on all PRs

* Don't use env var for adapter->version map

* Try to use " instead of '

* Try another workaround to launch the proper database service

* More temporary changes from pull_request_target to pull_request

* Attempt to set system variables for mariadb & mysql5

* Clean up database service selection

* Change pull_request back to pull_request_target

* Update changelog

* Copy Makefile test infrastructure from `dbt-snowflake` and update docs

* Fix test README link
  • Loading branch information
mwallace582 authored Jan 8, 2024
1 parent 37589c1 commit 22de9fe
Show file tree
Hide file tree
Showing 19 changed files with 792 additions and 219 deletions.
94 changes: 0 additions & 94 deletions .circleci/config.yml

This file was deleted.

16 changes: 16 additions & 0 deletions .flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
[flake8]
select =
E
W
F
ignore =
# makes Flake8 work like black
W503,
W504,
# makes Flake8 work like black
E203,
E741,
E501,
exclude = tests
per-file-ignores =
*/__init__.py: F401
3 changes: 3 additions & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# This codeowners file is used to ensure all PRs require reviews from the appropriate users.

* @dbeatty10 @mwallace582
105 changes: 105 additions & 0 deletions .github/scripts/integration-test-matrix.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
module.exports = ({ context }) => {
const defaultPythonVersion = "3.8";
const supportedPythonVersions = ["3.8", "3.9"];
const supportedAdapters = ["mysql", "mysql5", "mariadb"];
const adapterImages = {
mysql: "mysql:8.0",
mysql5: "mysql:5.7",
mariadb: "mariadb:10.5",
};

// if PR, generate matrix based on files changed and PR labels
if (context.eventName.includes("pull_request")) {
// `changes` is a list of adapter names that have related
// file changes in the PR
// ex: ['postgres', 'snowflake']
const changes = JSON.parse(process.env.CHANGES);
const labels = context.payload.pull_request.labels.map(({ name }) => name);
console.log("labels", labels);
console.log("changes", changes);
const testAllLabel = labels.includes("test all");
const include = [];

for (const adapter of supportedAdapters) {
if (
changes.includes(adapter) ||
testAllLabel ||
labels.includes(`test ${adapter}`)
) {
for (const pythonVersion of supportedPythonVersions) {
if (
pythonVersion === defaultPythonVersion ||
labels.includes(`test python${pythonVersion}`) ||
testAllLabel
) {
// always run tests on ubuntu by default
include.push({
os: "ubuntu-latest",
adapter,
"database-image": adapterImages[adapter],
"python-version": pythonVersion,
});

if (labels.includes("test windows") || testAllLabel) {
include.push({
os: "windows-latest",
adapter,
"database-image": adapterImages[adapter],
"python-version": pythonVersion,
});
}

if (labels.includes("test macos") || testAllLabel) {
include.push({
os: "macos-latest",
adapter,
"database-image": adapterImages[adapter],
"python-version": pythonVersion,
});
}
}
}
}
}

console.log("matrix", { include });

return {
include,
};
}
// if not PR, generate matrix of python version, adapter, and operating
// system to run integration tests on

const include = [];
// run for all adapters and python versions on ubuntu
for (const adapter of supportedAdapters) {
for (const pythonVersion of supportedPythonVersions) {
include.push({
os: 'ubuntu-latest',
adapter: adapter,
"database-image": adapterImages[adapter],
"python-version": pythonVersion,
});
}
}

// additionally include runs for all adapters, on macos and windows,
// but only for the default python version
for (const adapter of supportedAdapters) {
for (const operatingSystem of ["windows-latest", "macos-latest"]) {
include.push({
os: operatingSystem,
adapter: adapter,
"database-image": adapterImages[adapter],
"python-version": defaultPythonVersion,
});
}
}

console.log("matrix", { include });

return {
include,
};
};
20 changes: 20 additions & 0 deletions .github/scripts/update_dbt_core_branch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash -e
set -e

git_branch=$1
target_req_file="dev-requirements.txt"
core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g"
postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g"
tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g"
if [[ "$OSTYPE" == darwin* ]]; then
# mac ships with a different version of sed that requires a delimiter arg
sed -i "" "$core_req_sed_pattern" $target_req_file
sed -i "" "$postgres_req_sed_pattern" $target_req_file
sed -i "" "$tests_req_sed_pattern" $target_req_file
else
sed -i "$core_req_sed_pattern" $target_req_file
sed -i "$postgres_req_sed_pattern" $target_req_file
sed -i "$tests_req_sed_pattern" $target_req_file
fi
core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2)
bumpversion --allow-dirty --new-version "$core_version" major
Loading

0 comments on commit 22de9fe

Please sign in to comment.