diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index ee12485bf..222eff237 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -13,7 +13,7 @@ body: attributes: label: Singer SDK Version description: Version of the library you are using - placeholder: "0.39.1" + placeholder: "0.40.0" validations: required: true - type: checkboxes @@ -29,13 +29,12 @@ body: label: Python Version description: Version of Python you are using options: - - "3.6 (EOL)" - - "3.7 (EOL)" - - "3.8" - - "3.9" - - "3.10" - - "3.11" - "3.12" + - "3.11" + - "3.10" + - "3.9" + - "3.8" + - "3.7 or earlier" - "NA" validations: required: true @@ -78,3 +77,9 @@ body: render: Python validations: required: false + - type: input + id: slack_or_linen + attributes: + label: Link to Slack/Linen + description: Provide a link to the Slack or Linen conversation, if applicable + placeholder: "https://..." diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt index e6615dcc3..52415d85b 100644 --- a/.github/workflows/constraints.txt +++ b/.github/workflows/constraints.txt @@ -1,4 +1,4 @@ -griffe==0.48.0 +griffe==1.2.0 pip==24.2 poetry==1.8.3 poetry-plugin-export==1.8.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 712f2e3db..3974249d0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,7 @@ on: jobs: build: + name: Build artifacts runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -12,8 +13,44 @@ jobs: fetch-depth: 0 - uses: hynek/build-and-inspect-python-package@v2 + check-tag: + name: Check tag + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + outputs: + is_final: ${{ steps.check.outputs.is_final }} + steps: + - name: Check if tag is a pre-release + id: check + run: | + echo "is_final=$(echo '${{ github.ref }}' | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+$' && echo 'true' || echo 'false')" >> $GITHUB_OUTPUT + + provenance: + name: Provenance + runs-on: ubuntu-latest + needs: [build] + if: startsWith(github.ref, 'refs/tags/') + permissions: + id-token: write # Needed for attestations + attestations: write # Needed for attestations + outputs: + bundle-path: ${{ steps.attest.outputs.bundle-path }} + steps: + - uses: actions/download-artifact@v4 + with: + name: Packages + path: dist + - uses: actions/attest-build-provenance@v1 + id: attest + with: + subject-path: "./dist/singer_sdk*" + - uses: actions/upload-artifact@v4 + with: + name: Attestations + path: ${{ steps.attest.outputs.bundle-path }} + publish: - name: Publish to PyPI + name: PyPI runs-on: ubuntu-latest needs: [build] environment: @@ -28,23 +65,27 @@ jobs: name: Packages path: dist - name: Publish - uses: pypa/gh-action-pypi-publish@v1.9.0 + uses: pypa/gh-action-pypi-publish@v1.10.0 upload-to-release: name: Upload files to release runs-on: ubuntu-latest - needs: [build] - if: startsWith(github.ref, 'refs/tags/') + needs: [build, check-tag, provenance] + if: ${{ startsWith(github.ref, 'refs/tags/') && needs.check-tag.outputs.is_final == 'true' }} permissions: contents: write # Needed for uploading files to the release - id-token: write # Needed for attestations - attestations: write # Needed for attestations steps: - uses: actions/download-artifact@v4 with: name: Packages path: dist + + - uses: actions/download-artifact@v4 + with: + name: Attestations + path: attestations + - name: Upload wheel and sdist to release uses: svenstaro/upload-release-action@v2 with: @@ -52,14 +93,11 @@ jobs: tag: ${{ github.ref }} overwrite: true file_glob: true - - uses: actions/attest-build-provenance@v1 - id: attest - with: - subject-path: "./dist/singer_sdk*" + - name: Upload attestations to release uses: svenstaro/upload-release-action@v2 with: - file: ${{ steps.attest.outputs.bundle-path }} + file: attestations/attestation.jsonl tag: ${{ github.ref }} overwrite: true asset_name: attestations.intoto.jsonl diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3acc509fb..cf0602a29 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,6 +99,7 @@ jobs: - uses: actions/upload-artifact@v4 if: always() && (matrix.session == 'tests') with: + include-hidden-files: true name: coverage-data-nox_${{ matrix.session }}-${{ matrix.os }}-py${{ matrix.python-version }}_sqlalchemy_${{ matrix.sqlalchemy }} path: ".coverage.*" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e3bfd2a9f..22a0b74c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,14 +43,14 @@ repos: )$ - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.1 + rev: 0.29.2 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.7 + rev: v0.6.4 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] diff --git a/CHANGELOG.md b/CHANGELOG.md index 27d538a6d..46715c382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,40 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.40.0 (2024-09-02) + +### ✨ New + +- [#2486](https://github.com/meltano/sdk/issues/2486) Emit target metrics +- [#2567](https://github.com/meltano/sdk/issues/2567) A new `schema_is_valid` built-in tap test validates stream schemas against the JSON Schema specification +- [#2598](https://github.com/meltano/sdk/issues/2598) Stream map expressions now have access to the `Faker` class, rather than just a faker instance +- [#2549](https://github.com/meltano/sdk/issues/2549) Added a default user agent for REST and GraphQL taps + +### 🐛 Fixes + +- [#2613](https://github.com/meltano/sdk/issues/2613) Mismatch between timezone-aware and naive datetimes in start date and bookmarks is now correctly handled + +### ⚙️ Under the Hood + +- [#2628](https://github.com/meltano/sdk/issues/2628) Use context manager to read gzip batch files +- [#2619](https://github.com/meltano/sdk/issues/2619) Default to UTC when parsing dates without a known timezone +- [#2603](https://github.com/meltano/sdk/issues/2603) Backwards-compatible identifier quoting in fully qualified names +- [#2601](https://github.com/meltano/sdk/issues/2601) Improved SQL identifier (de)normalization +- [#2599](https://github.com/meltano/sdk/issues/2599) Remove `pytest-durations` dependency from `testing` and use native pytest option `--durations` +- [#2597](https://github.com/meltano/sdk/issues/2597) Mark pagination classes with `@override` decorator +- [#2596](https://github.com/meltano/sdk/issues/2596) Made `auth_headers` and `auth_params` of `APIAuthenticatorBase` simple instance attributes instead of decorated properties + +### 📚 Documentation Improvements + +- [#2639](https://github.com/meltano/sdk/issues/2639) Documented versions where `fake` and `Faker` objects were added to the stream maps context +- [#2629](https://github.com/meltano/sdk/issues/2629) Reference `get_starting_timestamp` in incremental replication guide +- [#2604](https://github.com/meltano/sdk/issues/2604) Update project sample links +- [#2595](https://github.com/meltano/sdk/issues/2595) Documented examples of stream glob expressions and property aliasing + +### 📦 Packaging changes + +- [#2640](https://github.com/meltano/sdk/issues/2640) Remove upper constraint on `faker` extra + ## v0.39.1 (2024-08-07) ### 🐛 Fixes diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml index 2bbeebf75..1399f9f8c 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml @@ -3,7 +3,26 @@ name: Test {{cookiecutter.mapper_id}} -on: [push] +on: + push: + branches: [main] + paths: + - .github/workflows/test.yml + - {{ cookiecutter.library_name }}/** + - tests/** + - poetry.lock + - pyproject.toml + - tox.ini + pull_request: + branches: [main] + paths: + - .github/workflows/test.yml + - {{ cookiecutter.library_name }}/** + - tests/** + - poetry.lock + - pyproject.toml + - tox.ini + workflow_dispatch: jobs: pytest: @@ -13,7 +32,12 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" steps: - uses: actions/checkout@v4 - name: Set up Python {{ '${{ matrix.python-version }}' }} diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml index 29308ec2d..9e35a5cbe 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml @@ -18,19 +18,19 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.1 + rev: 0.29.2 hooks: - id: check-dependabot - id: check-github-workflows - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.4 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.1 + rev: v1.11.2 hooks: - id: mypy diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml index e77cf5346..b397face4 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -31,12 +31,12 @@ packages = [ [tool.poetry.dependencies] python = ">=3.8" -singer-sdk = { version="~=0.39.1"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} } +singer-sdk = { version="~=0.40.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} } fs-s3fs = { version = "~=1.1.1", optional = true } [tool.poetry.group.dev.dependencies] pytest = ">=8" -singer-sdk = { version="~=0.39.1", extras = ["testing"] } +singer-sdk = { version="~=0.40.0", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml index a6a631c2c..85aba9d26 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml @@ -3,7 +3,26 @@ name: Test {{cookiecutter.tap_id}} -on: [push] +on: + push: + branches: [main] + paths: + - .github/workflows/test.yml + - {{ cookiecutter.library_name }}/** + - tests/** + - poetry.lock + - pyproject.toml + - tox.ini + pull_request: + branches: [main] + paths: + - .github/workflows/test.yml + - {{ cookiecutter.library_name }}/** + - tests/** + - poetry.lock + - pyproject.toml + - tox.ini + workflow_dispatch: jobs: pytest: @@ -13,7 +32,12 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" steps: - uses: actions/checkout@v4 - name: Set up Python {{ '${{ matrix.python-version }}' }} diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml index ced959bcc..45093ce09 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml @@ -18,20 +18,20 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.1 + rev: 0.29.2 hooks: - id: check-dependabot - id: check-github-workflows - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.4 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.1 + rev: v1.11.2 hooks: - id: mypy additional_dependencies: diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index 98851d925..3037bcabf 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -31,7 +31,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.8" importlib-resources = { version = "==6.4.*", python = "<3.9" } -singer-sdk = { version="~=0.39.1", extras = [ +singer-sdk = { version="~=0.40.0", extras = [ {%- if cookiecutter.auth_method == "JWT" -%}"jwt", {% endif -%} {%- if cookiecutter.faker_extra -%}"faker",{%- endif -%} ] } @@ -43,9 +43,9 @@ requests = "~=2.32.3" [tool.poetry.group.dev.dependencies] pytest = ">=8" {%- if cookiecutter.auth_method == "JWT" %} -singer-sdk = { version="~=0.39.1", extras = ["jwt", "testing"] } +singer-sdk = { version="~=0.40.0", extras = ["jwt", "testing"] } {%- else %} -singer-sdk = { version="~=0.39.1", extras = ["testing"] } +singer-sdk = { version="~=0.40.0", extras = ["testing"] } {%- endif %} [tool.poetry.extras] diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py index df3f9f754..74c8927e9 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py @@ -50,6 +50,16 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type == default="https://api.mysample.com", description="The url for the API service", ), + {%- if cookiecutter.stream_type in ("GraphQL", "REST") %} + th.Property( + "user_agent", + th.StringType, + description=( + "A custom User-Agent header to send with each request. Default is " + "'/'" + ), + ), + {%- endif %} ).to_dict() {%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %} diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml index fda907e07..2ed7a8bc5 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml @@ -3,7 +3,26 @@ name: Test {{cookiecutter.target_id}} -on: [push] +on: + push: + branches: [main] + paths: + - .github/workflows/test.yml + - {{ cookiecutter.library_name }}/** + - tests/** + - poetry.lock + - pyproject.toml + - tox.ini + pull_request: + branches: [main] + paths: + - .github/workflows/test.yml + - {{ cookiecutter.library_name }}/** + - tests/** + - poetry.lock + - pyproject.toml + - tox.ini + workflow_dispatch: jobs: pytest: @@ -13,7 +32,12 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" steps: - uses: actions/checkout@v4 - name: Set up Python {{ '${{ matrix.python-version }}' }} diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml b/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml index 3cab6f92b..ee72f1515 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/.pre-commit-config.yaml @@ -18,20 +18,20 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.1 + rev: 0.29.2 hooks: - id: check-dependabot - id: check-github-workflows - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.4 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.1 + rev: v1.11.2 hooks: - id: mypy additional_dependencies: diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/README.md b/cookiecutter/target-template/{{cookiecutter.target_id}}/README.md index 983be1ce5..6733f0fc0 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/README.md +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/README.md @@ -51,7 +51,7 @@ This Singer target will automatically import any environment variables within th `.env` if the `--config=ENV` is provided, such that config values will be considered if a matching environment variable is set either in the terminal context or in the `.env` file. -### Source Authentication and Authorization +### Authentication and Authorization