-
Notifications
You must be signed in to change notification settings - Fork 1
184 lines (182 loc) · 8.16 KB
/
publish_transformation.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
name: Publish transformation
on:
push:
tags:
- "transformation-*-v*.*.*"
env:
SNOW_USER: ${{ secrets.SNOW_USER }}
SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }}
# DBT assumes the account is in the form of <account>.<region>
SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}"
SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }}
SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }}
SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }}
SNOW_REGION: ${{ secrets.SNOW_REGION }}
CLOUDQUERY_API_KEY: ${{ secrets.CLOUDQUERY_API_KEY }}
jobs:
prepare:
runs-on: ubuntu-latest
outputs:
transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }}
transformation_version: ${{ fromJson(steps.set-result.outputs.result).transformation_version }}
zipPath: ${{ fromJson(steps.set-result.outputs.result).zipPath }}
postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }}
snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }}
bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }}
s3: ${{ fromJson(steps.set-result.outputs.result).s3 }}
clickhouse: ${{ fromJson(steps.set-result.outputs.result).clickhouse }}
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/github-script@v7
id: set-result
env:
TRANSFORMATION_TAG: ${{ github.ref_name }}
with:
script: |
const fs = require('fs/promises');
const { TRANSFORMATION_TAG } = process.env;
const [_, topLevelDir, ...rest] = TRANSFORMATION_TAG.split('-');
const version = rest.pop();
const transformation_dir = `transformations/${topLevelDir}/${rest.join('-')}`;
const [postgres, snowflake, bigquery, s3, clickhouse] = await Promise.all([
fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
fs.access(`${transformation_dir}/tests/s3.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
fs.access(`${transformation_dir}/tests/clickhouse.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
]);
return {
transformation_dir,
transformation_version: version,
postgres,
snowflake,
bigquery,
s3,
clickhouse,
};
publish-transformation:
permissions:
id-token: 'write'
contents: 'read'
needs: prepare
runs-on: ubuntu-latest
services:
postgres:
image: postgres:11
env:
POSTGRES_PASSWORD: pass
POSTGRES_USER: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Checkout
uses: actions/checkout@v4
# Required for BigQuery targets
- name: Authenticate to Google Cloud
uses: 'google-github-actions/auth@v2'
with:
workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider'
service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com'
# Required for athena targets
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::590184131402:role/cq-integration-tests-aws-github-action
aws-region: us-east-1
- name: Update Release Notes
uses: actions/github-script@v7
env:
TRANSFORMATION_DIR: ${{ needs.prepare.outputs.transformation_dir }}
with:
script: |
const fs = require('fs/promises');
const { TRANSFORMATION_DIR } = process.env;
const { data } = await github.rest.repos.getReleaseByTag({
owner: "cloudquery",
repo: context.repo.repo,
tag: context.ref.replace('refs/tags/', ''),
});
const releaseNotes = data.body;
const linksRemoved = releaseNotes.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1');
const changelogPath = `${TRANSFORMATION_DIR}/changelog.md`;
await fs.writeFile(changelogPath, linksRemoved);
- uses: actions/setup-python@v5
with:
python-version: "3.9"
cache: "pip"
cache-dependency-path: "./${{ needs.prepare.outputs.transformation_dir }}/requirements.txt"
- name: Install dependencies
working-directory: ${{ needs.prepare.outputs.transformation_dir }}
run: pip install -r requirements.txt
- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.12.10
- name: Migrate DB Postgres
if: needs.prepare.outputs.postgres == 'true'
run: cloudquery migrate tests/postgres.yml
working-directory: ${{ needs.prepare.outputs.transformation_dir }}
env:
CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
- name: Migrate DB Snowflake
if: needs.prepare.outputs.snowflake == 'true'
run: cloudquery migrate tests/snowflake.yml
working-directory: ${{ needs.prepare.outputs.transformation_dir }}
env:
SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}"
- name: Migrate DB BigQuery
if: needs.prepare.outputs.bigquery == 'true'
run: cloudquery migrate tests/bigquery.yml
working-directory: ${{ needs.prepare.outputs.transformation_dir }}
- name: Migrate DB Athena
if: needs.prepare.outputs.s3 == 'true'
run: cloudquery migrate tests/s3.yml
working-directory: ${{ needs.prepare.outputs.transformation_dir }}
- name: Start ClickHouse Container
if: needs.prepare.outputs.clickhouse == 'true'
run: |
docker run --platform linux/amd64 -d --name clickhouse-server --rm -p 8123:8123 -p 9000:9000 \
-e CLICKHOUSE_PASSWORD=test \
-e CLICKHOUSE_USER=cq \
-e CLICKHOUSE_DB=cloudquery \
-e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
-v ${{ github.workspace }}/.github/clickhouse.xml:/etc/clickhouse-server/users.d/cloudquery.xml \
clickhouse/clickhouse-server:22.1.2
sudo apt update && sudo apt install wait-for-it -y
wait-for-it -h localhost -p 9000
- name: Migrate DB Clickhouse
if: needs.prepare.outputs.clickhouse == 'true'
working-directory: ${{ needs.prepare.outputs.transformation_dir }}
env:
CLICKHOUSE_CONNECTION_STRING: "clickhouse://cq:test@localhost:9000/cloudquery"
run: cloudquery migrate tests/clickhouse.yml
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 'lts/*'
cache: 'npm'
cache-dependency-path: scripts/dbt-pack/package-lock.json
- name: Install Dependencies
run: npm ci
working-directory: ./scripts/dbt-pack
- name: Pack DBT
working-directory: ./scripts/dbt-pack
run: node index.js dbt-pack --project-dir=../../${{ needs.prepare.outputs.transformation_dir }}
- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.12.10
- name: Publish tranformation
working-directory: ${{ needs.prepare.outputs.transformation_dir }}
env:
CLOUDQUERY_API_KEY: ${{ secrets.CLOUDQUERY_API_KEY }}
run: |
cloudquery addon publish manifest.json ${{ needs.prepare.outputs.transformation_version }} -f