diff --git a/README.md b/README.md index cc1473b..afc255d 100644 --- a/README.md +++ b/README.md @@ -79,10 +79,10 @@ jobs: with: github-token: ${{ secrets.MY_PAT }} ``` -## BigQuery +## Snowflake -The BigQuery action depends on Google's python BigQuery library. -More information can be found about specific connection credentials in Google's documentation [here](https://cloud.google.com/python/docs/reference/bigquery/latest). +The Snowflake action depends on Snowflake's python connector library. +You can find complete documentation about the library in the Snowflake docs [here](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector) with more detail about the connector [here](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api). ### Fields @@ -91,9 +91,13 @@ More information can be found about specific connection credentials in Google's | Field | Required | Default | Description | |-----|-----|-----|-----| -| project | yes | | The BigQuery project string | -| dataset | yes | | The BigQuery dataset string | -| credentials | yes | | A JSON credential string for use with google oauth service account [connections](https://google-auth.readthedocs.io/en/master/reference/google.oauth2.service_account.html#google.oauth2.service_account.Credentials) | +| db-user | yes | | The database user | +| db-password | yes | | The database password | +| account | yes | | Associated Snowflake account | +| warehouse | yes | | Associated Snowflake warehouse | +| role | no | | Optional Snowflake role | +| database | no | | Optional Snowflake database | +| schema | no | | Optional snowflake schema | @@ -105,39 +109,34 @@ More information can be found about specific connection credentials in Google's ```yaml copy on: - pull_request -name: BigQuery +name: Snowflake jobs: - test_bigquery: + test_snowflake: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Run Grai Action - uses: grai-io/grai-actions/bigquery@master + uses: grai-io/grai-actions/snowflake@master with: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io - project: my-bigquery-project - dataset: my-bigquery-dataset - credentials: '{ "type": "service_account", "project_id": "demo", "private_key_id": - "your_private_key_id", "private_key": "your_private_key", "client_email": - "your@email.iam.gserviceaccount.com", "client_id": "your_client_id", "auth_uri": - "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/you%40email.iam.gserviceaccount.com" - }' + db-user: my-user + db-password: my-password + account: my-account + warehouse: my-warehouse ``` -## Snowflake +## Redshift -The Snowflake action depends on Snowflake's python connector library. -You can find complete documentation about the library in the Snowflake docs [here](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector) with more detail about the connector [here](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api). +The Redshift action depends on Amazon's python connector library. +You can find complete documentation about the library in the AWS docs [here](https://github.com/aws/amazon-redshift-python-driver). ### Fields @@ -146,13 +145,11 @@ You can find complete documentation about the library in the Snowflake docs [her | Field | Required | Default | Description | |-----|-----|-----|-----| +| db-host | yes | | The database host | +| db-port | no | 5439 | The database port | +| db-database-name | yes | | The database name | | db-user | yes | | The database user | | db-password | yes | | The database password | -| account | yes | | Associated Snowflake account | -| warehouse | yes | | Associated Snowflake warehouse | -| role | no | | Optional Snowflake role | -| database | no | | Optional Snowflake database | -| schema | no | | Optional snowflake schema | @@ -164,36 +161,35 @@ You can find complete documentation about the library in the Snowflake docs [her ```yaml copy on: - pull_request -name: Snowflake +name: Redshift jobs: - test_snowflake: + test_redshift: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Run Grai Action - uses: grai-io/grai-actions/snowflake@master + uses: grai-io/grai-actions/redshift@master with: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io - db-user: my-user - db-password: my-password - account: my-account - warehouse: my-warehouse + db-host: redshift-cluster-1.abc123xyz789.us-east-1.redshift.amazonaws.com + db-port: '5439' + db-database-name: dev + db-user: admin + db-password: password ``` -## Flat File - -The flat file action reads a flat file like csv, parquet, or feather inside of your github project to perform tests and update your grai instance. -Because of this, it's critical your file is up to date on each pull request. +## MySQL -Make sure to include an `- uses: actions/checkout@v3` step in your workflow so that your repo code is available. +The MySQL action depends on the python mysql library. +You can find complete documentation about the library [here](https://dev.mysql.com/doc/connector-python). ### Fields @@ -202,7 +198,11 @@ Make sure to include an `- uses: actions/checkout@v3` step in your workflow so t | Field | Required | Default | Description | |-----|-----|-----|-----| -| file | yes | | Local file to track with Grai | +| db-host | yes | | The database host | +| db-port | no | 3306 | The database port | +| db-database-name | yes | | The database name | +| db-user | yes | | The database user | +| db-password | yes | | The database password | @@ -214,36 +214,37 @@ Make sure to include an `- uses: actions/checkout@v3` step in your workflow so t ```yaml copy on: - pull_request -name: Flat File +name: MySQL jobs: - test_flat-file: + test_mysql: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Run Grai Action - uses: grai-io/grai-actions/flat-file@master + uses: grai-io/grai-actions/mysql@master with: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io - file: ./tests/flat-file/low-numbers.csv + db-host: dev.mysql.com + db-port: '3306' + db-database-name: my_db + db-user: my_user + db-password: my_password ``` -## dbt - -The dbt action reads a manifest.json file inside of your github project to perform tests and update your grai instance. -Because of this, it's critical your manifest.json file is up to date on each pull request. -One way to do this is to perform `dbt build` as part of your CI action but there are multiple ways to generate an up-to-date manifest file. +## Flat File -* More information about manifest.json files can be found [here](https://docs.getdbt.com/reference/artifacts/manifest-json). -* Make sure to include an `- uses: actions/checkout@v3` step in your workflow so that your repo code is available. +The flat file action reads a flat file like csv, parquet, or feather inside of your github project to perform tests and update your grai instance. +Because of this, it's critical your file is up to date on each pull request. +Make sure to include an `- uses: actions/checkout@v3` step in your workflow so that your repo code is available. ### Fields @@ -252,7 +253,7 @@ One way to do this is to perform `dbt build` as part of your CI action but there | Field | Required | Default | Description | |-----|-----|-----|-----| -| manifest-file | yes | | The file location in your repository of the updated manifest.json file | +| file | yes | | Local file to track with Grai | @@ -264,31 +265,31 @@ One way to do this is to perform `dbt build` as part of your CI action but there ```yaml copy on: - pull_request -name: dbt +name: Flat File jobs: - test_dbt: + test_flat-file: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Run Grai Action - uses: grai-io/grai-actions/dbt@master + uses: grai-io/grai-actions/flat-file@master with: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io - manifest-file: ./tests/dbt/manifest.json + file: ./tests/flat-file/low-numbers.csv ``` -## MySQL +## BigQuery -The MySQL action depends on the python mysql library. -You can find complete documentation about the library [here](https://dev.mysql.com/doc/connector-python). +The BigQuery action depends on Google's python BigQuery library. +More information can be found about specific connection credentials in Google's documentation [here](https://cloud.google.com/python/docs/reference/bigquery/latest). ### Fields @@ -297,11 +298,9 @@ You can find complete documentation about the library [here](https://dev.mysql.c | Field | Required | Default | Description | |-----|-----|-----|-----| -| db-host | yes | | The database host | -| db-port | no | 3306 | The database port | -| db-database-name | yes | | The database name | -| db-user | yes | | The database user | -| db-password | yes | | The database password | +| project | yes | | The BigQuery project string | +| dataset | yes | | The BigQuery dataset string | +| credentials | yes | | A JSON credential string for use with google oauth service account [connections](https://google-auth.readthedocs.io/en/master/reference/google.oauth2.service_account.html#google.oauth2.service_account.Credentials) | @@ -313,37 +312,40 @@ You can find complete documentation about the library [here](https://dev.mysql.c ```yaml copy on: - pull_request -name: MySQL +name: BigQuery jobs: - test_mysql: + test_bigquery: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Run Grai Action - uses: grai-io/grai-actions/mysql@master + uses: grai-io/grai-actions/bigquery@master with: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io - db-host: dev.mysql.com - db-port: '3306' - db-database-name: my_db - db-user: my_user - db-password: my_password + project: my-bigquery-project + dataset: my-bigquery-dataset + credentials: '{ "type": "service_account", "project_id": "demo", "private_key_id": + "your_private_key_id", "private_key": "your_private_key", "client_email": + "your@email.iam.gserviceaccount.com", "client_id": "your_client_id", "auth_uri": + "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/you%40email.iam.gserviceaccount.com" + }' ``` -## Fivetran +## Postgres -The Fivetran Action relies upon access to Fivetran's API endpoint. -This endpoint is configurable if you have a non-standard implementation but should generally be left alone. +The Postgres action depends on the python psycopg2 library. +You can find complete documentation about the library [here](https://www.psycopg.org/docs/). -Authentication with their services will require an API key and secret but you can find more documentation about generating these values [here](https://fivetran.com/docs/rest-api/getting-started#instructions). ### Fields @@ -351,39 +353,15 @@ Authentication with their services will require an API key and secret but you ca | Field | Required | Default | Description | |-----|-----|-----|-----| -| fivetran-endpoint | no | https://api.fivetran.com/v1 | Fivetran API endpoint | -| fivetran-api-key | yes | | Your Fivetran user api key | -| fivetran-api-secret | yes | | Your Fivetran user api secret | -| namespace-map | no | | A JSON string containing a mapping between Fivetran connections and Grai namespaces | +| db-host | yes | | The database host | +| db-port | no | 5432 | The database port | +| db-database-name | yes | | The database name | +| db-user | yes | | The database user | +| db-password | yes | | The database password | -The `namespace` field in the Fivetran Action works slightly differently than other action. -It is used as a default namespace for all connections not specified in the `namespace_map`. -You can find more information about that below. - - -#### Namespace Map - -Each Fivetran connection has a connector id and synchronizes from a source to a sync. -For example, a sync from your production database to data warehouse would have an associated connector id. - -Because the Fivetran Action synchronizes from all of your Fivetran connections it uses the `namespace_map` value to know which connectors belong to which Grai namespaces. -The namespace map should be a JSON string with the Grai namespace for each source and destination of each connector id e.g. - -```json -{ - "": { - "source": "", - "destination", "" - } -} -``` - - -You can find connector id's for all of your Fivetran connections in the [API](https://fivetran.com/docs/rest-api/faq/find-connector_id) - ### Example @@ -391,34 +369,40 @@ You can find connector id's for all of your Fivetran connections in the [API](ht ```yaml copy on: - pull_request -name: Fivetran +name: PostgreSQL jobs: - test_fivetran: + test_postgres: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Run Grai Action - uses: grai-io/grai-actions/fivetran@master + uses: grai-io/grai-actions/postgres@master with: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io - fivetran-api-key: hHqP5c2nIY0B6fpa - fivetran-api-secret: 1234567890abcdef1234567890abcdef - namespace-map: '{"operative_combination": {"source": "source_namespace", - "destination": "destination_namespace"}}' + db-host: prod.db.com + db-port: '5432' + db-database-name: my_database + db-user: my_user + db-password: my_password ``` -## Redshift +## dbt + +The dbt action reads a manifest.json file inside of your github project to perform tests and update your grai instance. +Because of this, it's critical your manifest.json file is up to date on each pull request. +One way to do this is to perform `dbt build` as part of your CI action but there are multiple ways to generate an up-to-date manifest file. + +* More information about manifest.json files can be found [here](https://docs.getdbt.com/reference/artifacts/manifest-json). +* Make sure to include an `- uses: actions/checkout@v3` step in your workflow so that your repo code is available. -The Redshift action depends on Amazon's python connector library. -You can find complete documentation about the library in the AWS docs [here](https://github.com/aws/amazon-redshift-python-driver). ### Fields @@ -427,11 +411,7 @@ You can find complete documentation about the library in the AWS docs [here](htt | Field | Required | Default | Description | |-----|-----|-----|-----| -| db-host | yes | | The database host | -| db-port | no | 5439 | The database port | -| db-database-name | yes | | The database name | -| db-user | yes | | The database user | -| db-password | yes | | The database password | +| manifest-file | yes | | The file location in your repository of the updated manifest.json file | @@ -443,36 +423,33 @@ You can find complete documentation about the library in the AWS docs [here](htt ```yaml copy on: - pull_request -name: Redshift +name: dbt jobs: - test_redshift: + test_dbt: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Run Grai Action - uses: grai-io/grai-actions/redshift@master + uses: grai-io/grai-actions/dbt@master with: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io - db-host: redshift-cluster-1.abc123xyz789.us-east-1.redshift.amazonaws.com - db-port: '5439' - db-database-name: dev - db-user: admin - db-password: password + manifest-file: ./tests/dbt/manifest.json ``` -## Postgres +## Fivetran -The Postgres action depends on the python psycopg2 library. -You can find complete documentation about the library [here](https://www.psycopg.org/docs/). +The Fivetran Action relies upon access to Fivetran's API endpoint. +This endpoint is configurable if you have a non-standard implementation but should generally be left alone. +Authentication with their services will require an API key and secret but you can find more documentation about generating these values [here](https://fivetran.com/docs/rest-api/getting-started#instructions). ### Fields @@ -480,15 +457,39 @@ You can find complete documentation about the library [here](https://www.psycopg | Field | Required | Default | Description | |-----|-----|-----|-----| -| db-host | yes | | The database host | -| db-port | no | 5432 | The database port | -| db-database-name | yes | | The database name | -| db-user | yes | | The database user | -| db-password | yes | | The database password | +| fivetran-endpoint | no | https://api.fivetran.com/v1 | Fivetran API endpoint | +| fivetran-api-key | yes | | Your Fivetran user api key | +| fivetran-api-secret | yes | | Your Fivetran user api secret | +| namespace-map | no | | A JSON string containing a mapping between Fivetran connections and Grai namespaces | +The `namespace` field in the Fivetran Action works slightly differently than other action. +It is used as a default namespace for all connections not specified in the `namespace_map`. +You can find more information about that below. + + +#### Namespace Map + +Each Fivetran connection has a connector id and synchronizes from a source to a sync. +For example, a sync from your production database to data warehouse would have an associated connector id. + +Because the Fivetran Action synchronizes from all of your Fivetran connections it uses the `namespace_map` value to know which connectors belong to which Grai namespaces. +The namespace map should be a JSON string with the Grai namespace for each source and destination of each connector id e.g. + +```json +{ + "": { + "source": "", + "destination", "" + } +} +``` + + +You can find connector id's for all of your Fivetran connections in the [API](https://fivetran.com/docs/rest-api/faq/find-connector_id) + ### Example @@ -496,26 +497,25 @@ You can find complete documentation about the library [here](https://www.psycopg ```yaml copy on: - pull_request -name: PostgreSQL +name: Fivetran jobs: - test_postgres: + test_fivetran: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Run Grai Action - uses: grai-io/grai-actions/postgres@master + uses: grai-io/grai-actions/fivetran@master with: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io - db-host: prod.db.com - db-port: '5432' - db-database-name: my_database - db-user: my_user - db-password: my_password + fivetran-api-key: hHqP5c2nIY0B6fpa + fivetran-api-secret: 1234567890abcdef1234567890abcdef + namespace-map: '{"operative_combination": {"source": "source_namespace", + "destination": "destination_namespace"}}' ``` @@ -569,7 +569,7 @@ jobs: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io db-user: sa db-password: sa_password diff --git a/bigquery/README.md b/bigquery/README.md index f73254c..6a66e00 100644 --- a/bigquery/README.md +++ b/bigquery/README.md @@ -37,7 +37,7 @@ jobs: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io project: my-bigquery-project dataset: my-bigquery-dataset diff --git a/dbt/README.md b/dbt/README.md index fed7282..a385f0f 100644 --- a/dbt/README.md +++ b/dbt/README.md @@ -40,7 +40,7 @@ jobs: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io manifest-file: ./tests/dbt/manifest.json diff --git a/fivetran/README.md b/fivetran/README.md index 78339a4..a14d6d6 100644 --- a/fivetran/README.md +++ b/fivetran/README.md @@ -64,7 +64,7 @@ jobs: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io fivetran-api-key: hHqP5c2nIY0B6fpa fivetran-api-secret: 1234567890abcdef1234567890abcdef diff --git a/flat-file/README.md b/flat-file/README.md index 3410a9d..631924c 100644 --- a/flat-file/README.md +++ b/flat-file/README.md @@ -37,7 +37,7 @@ jobs: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io file: ./tests/flat-file/low-numbers.csv diff --git a/mssql/README.md b/mssql/README.md index 7b40e2a..5fdcae3 100644 --- a/mssql/README.md +++ b/mssql/README.md @@ -46,7 +46,7 @@ jobs: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io db-user: sa db-password: sa_password diff --git a/mysql/README.md b/mysql/README.md index 8a278a5..a2c835a 100644 --- a/mysql/README.md +++ b/mysql/README.md @@ -39,7 +39,7 @@ jobs: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io db-host: dev.mysql.com db-port: '3306' diff --git a/postgres/README.md b/postgres/README.md index 614f120..9d0d8a6 100644 --- a/postgres/README.md +++ b/postgres/README.md @@ -39,7 +39,7 @@ jobs: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io db-host: prod.db.com db-port: '5432' diff --git a/redshift/README.md b/redshift/README.md index d0dc1f8..56a76cd 100644 --- a/redshift/README.md +++ b/redshift/README.md @@ -39,7 +39,7 @@ jobs: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io db-host: redshift-cluster-1.abc123xyz789.us-east-1.redshift.amazonaws.com db-port: '5439' diff --git a/snowflake/README.md b/snowflake/README.md index 187ca9e..dc406b9 100644 --- a/snowflake/README.md +++ b/snowflake/README.md @@ -41,7 +41,7 @@ jobs: namespace: my_apps_grai_namespace api-key: my_grai_api_key action: tests - source_name: prod-db + source-name: prod-db grai-api-url: https://api.grai.io db-user: my-user db-password: my-password