Merge branch 'main' into edgarrmondragon/refactor/msgspec-impl-naive

meltano · Jul 25, 2024 · f691f78 · f691f78
2 parents 8aefade + 433b7fe
commit f691f78
Show file tree

Hide file tree

Showing 69 changed files with 1,690 additions and 739 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -6,7 +6,7 @@ updates:
       interval: weekly
       time: "12:00"
     reviewers: [meltano/engineering]
-    labels: [deps]
+    labels: [Dependencies]
     groups:
       development-dependencies:
         dependency-type: development
@@ -21,7 +21,7 @@ updates:
       interval: weekly
       time: "12:00"
     reviewers: [meltano/engineering]
-    labels: [deps]
+    labels: [Dependencies]
     groups:
       ci:
         patterns:
@@ -31,7 +31,7 @@ updates:
     schedule:
       interval: weekly
     reviewers: [meltano/engineering]
-    labels: [deps]
+    labels: [Dependencies]
     groups:
       actions:
         patterns:

diff --git a/.github/semantic.yml b/.github/semantic.yml
@@ -15,6 +15,7 @@ types:
   - docs
   - feat
   - fix
+  - packaging
   - perf
   - refactor
   - revert

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -113,9 +113,6 @@ jobs:
       SAMPLE_TAP_GITLAB_GROUP_IDS: ${{ secrets.SAMPLE_TAP_GITLAB_GROUP_IDS }}
       SAMPLE_TAP_GITLAB_PROJECT_IDS: ${{ secrets.SAMPLE_TAP_GITLAB_PROJECT_IDS }}
       SAMPLE_TAP_GITLAB_START_DATE: "2022-01-01T00:00:00Z"
-      SAMPLE_TAP_GOOGLE_ANALYTICS_CLIENT_EMAIL: ${{ secrets.SAMPLE_TAP_GOOGLE_ANALYTICS_CLIENT_EMAIL }}
-      SAMPLE_TAP_GOOGLE_ANALYTICS_PRIVATE_KEY: ${{ secrets.SAMPLE_TAP_GOOGLE_ANALYTICS_PRIVATE_KEY }}
-      SAMPLE_TAP_GOOGLE_ANALYTICS_VIEW_ID: ${{ secrets.SAMPLE_TAP_GOOGLE_ANALYTICS_VIEW_ID }}
 
     steps:
     - uses: actions/checkout@v4

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -50,7 +50,7 @@ repos:
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.2
+  rev: v0.5.4
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.12"
 
 sphinx:
   builder: html

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml
@@ -13,18 +13,26 @@ plugins:
       streams:
       - stream_name: animals
         input_filename: https://raw.githubusercontent.com/meltano/tap-smoke-test/main/demo-data/animals-data.jsonl
+
   loaders:
   - name: target-jsonl
     variant: andyh1203
     pip_url: target-jsonl
+
   mappers:
   - name: "{{cookiecutter.mapper_id}}"
-    pip_url: -e .
     namespace: "{{cookiecutter.library_name}}"
-    # TODO: replace these with the actual settings
+    pip_url: -e .
+
+    # TODO: Declare settings and their types here:
     settings:
     - name: example_config
       kind: string
+      label: Example Config
+      description: An example configuration setting
+
+    # TODO: Declare mapping instances here:
+    # https://docs.meltano.com/guide/mappers/#example-1
     mappings:
     - name: example
       config:

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml
@@ -5,7 +5,7 @@ name = "{{cookiecutter.variant}}-{{cookiecutter.mapper_id}}"
 name = "{{cookiecutter.mapper_id}}"
 {%- endif %}
 version = "0.0.1"
-description = "`{{cookiecutter.mapper_id}}` is a Singer mapper {{cookiecutter.name}}, built with the Meltano Singer SDK."
+description = "Singer mapper {{cookiecutter.name}}, built with the Meltano Singer SDK."
 readme = "README.md"
 authors = ["{{ cookiecutter.admin_name }} <{{ cookiecutter.admin_email }}>"]
 keywords = [
@@ -35,7 +35,7 @@ singer-sdk = { version="~=0.38.0"{{ ', extras = ["faker"]' if cookiecutter.faker
 fs-s3fs = { version = "~=1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
-pytest = ">=7.4.0"
+pytest = ">=8"
 singer-sdk = { version="~=0.38.0", extras = ["testing"] }
 
 [tool.poetry.extras]

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/meltano.yml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/meltano.yml
@@ -15,16 +15,32 @@ plugins:
     - discover
     - about
     - stream-maps
-    config:
-      start_date: '2010-01-01T00:00:00Z'
+
+    # TODO: Declare settings and their types here:
     settings:
-    # TODO: To configure using Meltano, declare settings and their types here:
     - name: username
+      label: Username
+      description: The username to use for authentication
+
     - name: password
       kind: password
+      label: Password
+      description: The password to use for authentication
       sensitive: true
+
     - name: start_date
-      value: '2010-01-01T00:00:00Z'
+      kind: date_iso8601
+      label: Start Date
+      description: Initial date to start extracting data from
+
+    # TODO: Declare required settings here:
+    settings_group_validation:
+    - [username, password]
+
+    # TODO: Declare default configuration values here:
+    config:
+      start_date: '2010-01-01T00:00:00Z'
+
   loaders:
   - name: target-jsonl
     variant: andyh1203

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -5,7 +5,7 @@ name = "{{cookiecutter.variant}}-{{cookiecutter.tap_id}}"
 name = "{{cookiecutter.tap_id}}"
 {%- endif %}
 version = "0.0.1"
-description = "`{{cookiecutter.tap_id}}` is a Singer tap for {{cookiecutter.source_name}}, built with the Meltano Singer SDK."
+description = "Singer tap for {{cookiecutter.source_name}}, built with the Meltano Singer SDK."
 readme = "README.md"
 authors = ["{{ cookiecutter.admin_name }} <{{ cookiecutter.admin_email }}>"]
 keywords = [
@@ -37,11 +37,11 @@ singer-sdk = { version="~=0.38.0", extras = [
 ] }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
-requests = "~=2.32.0"
+requests = "~=2.32.3"
 {%- endif %}
 
 [tool.poetry.group.dev.dependencies]
-pytest = ">=7.4.0"
+pytest = ">=8"
 {%- if cookiecutter.auth_method == "JWT" %}
 singer-sdk = { version="~=0.38.0", extras = ["jwt", "testing"] }
 {%- else %}

diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/meltano.yml b/cookiecutter/target-template/{{cookiecutter.target_id}}/meltano.yml
@@ -5,7 +5,15 @@ default_environment: test
 environments:
 - name: test
 plugins:
-  extractors: []
+  extractors:
+  - name: tap-smoke-test
+    variant: meltano
+    pip_url: git+https://github.com/meltano/tap-smoke-test.git
+    config:
+      streams:
+      - stream_name: animals
+        input_filename: https://raw.githubusercontent.com/meltano/tap-smoke-test/main/demo-data/animals-data.jsonl
+
   loaders:
   - name: "{{cookiecutter.target_id}}"
     namespace: "{{cookiecutter.library_name}}"
@@ -14,13 +22,19 @@ plugins:
     - about
     - stream-maps
     - record-flattening
-    config:
-      start_date: '2010-01-01T00:00:00Z'
+
+    # TODO: Declare settings and their types here:
     settings:
-    # TODO: To configure using Meltano, declare settings and their types here:
     - name: username
+      label: Username
+      description: The username to use for authentication
+
     - name: password
       kind: password
+      label: Password
+      description: The password to use for authentication
       sensitive: true
-    - name: start_date
-      value: '2010-01-01T00:00:00Z'
+
+    # TODO: Declare required settings here:
+    settings_group_validation:
+    - [username, password]
diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml
@@ -5,7 +5,7 @@ name = "{{cookiecutter.variant}}-{{cookiecutter.target_id}}"
 name = "{{cookiecutter.target_id}}"
 {%- endif %}
 version = "0.0.1"
-description = "`{{cookiecutter.target_id}}` is a Singer target for {{cookiecutter.destination_name}}, built with the Meltano Singer SDK."
+description = "Singer target for {{cookiecutter.destination_name}}, built with the Meltano Singer SDK."
 readme = "README.md"
 authors = ["{{ cookiecutter.admin_name }} <{{ cookiecutter.admin_email }}>"]
 keywords = [
@@ -33,11 +33,11 @@ python = ">=3.8"
 singer-sdk = { version="~=0.38.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 {%- if cookiecutter.serialization_method != "SQL" %}
-requests = "~=2.32.0"
+requests = "~=2.32.3"
 {%- endif %}
 
 [tool.poetry.dev-dependencies]
-pytest = ">=7.4.0"
+pytest = ">=8"
 singer-sdk = { version="~=0.38.0", extras = ["testing"] }
 
 [tool.poetry.extras]

diff --git a/docs/implementation/logging.md b/docs/implementation/logging.md
@@ -77,3 +77,21 @@ This will send metrics to a `metrics.log`:
 2022-09-29 00:48:53,302 INFO METRIC: {"metric_type": "timer", "metric": "sync_duration", "value": 0.5258760452270508, "tags": {"stream": "countries", "context": {}, "status": "succeeded"}}
 2022-09-29 00:48:53,303 INFO METRIC: {"metric_type": "counter", "metric": "record_count", "value": 250, "tags": {"stream": "countries", "context": {}}}
 ```
+
+## For package developers
+
+If you're developing a tap or target package and would like to customize its logging configuration, you can put a `default_loggging.yml` file in the package root to set the default logging configuration for your package. This file will be used if the `SINGER_SDK_LOG_CONFIG` environment variable is not set:
+
+```
+.
+├── README.md
+├── poetry.lock
+├── pyproject.toml
+└── tap_example
+    ├── __init__.py
+    ├── __main__.py
+    ├── default_logging.yml  # <-- This file will be used if SINGER_SDK_LOG_CONFIG is not set
+    ├── client.py
+    ├── streams.py
+    └── tap.py
+```
diff --git a/docs/partitioning.md b/docs/partitioning.md
@@ -3,6 +3,10 @@
 The Tap SDK supports stream partitioning, meaning a set of substreams
 which each have their own state and their own distinct queryable domain.
 
+You can read more about state partitioning in the
+[State Implemetation](./implementation/state.md#partitioned-state) explanation
+document.
+
 ## If you do not require partitioning
 
 In general, developers can simply ignore the [`context`](./context_object.md) arguments

diff --git a/noxfile.py b/noxfile.py
@@ -186,7 +186,7 @@ def docs_serve(session: Session) -> None:
         "build",
         "-W",
     ]
-    session.install(".[docs]")
+    session.install(".[docs]", "sphinx-autobuild")
 
     build_dir = Path("build")
     if build_dir.exists():