From 5a6974445a8d2cec344680c91df4df8f9a301901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Wed, 31 Jan 2024 10:42:00 -0600 Subject: [PATCH] test: Test with singer-sdk prerelease --- .github/workflows/meltano-run.yml | 5 +++ meltano.yml | 9 ++++- poetry.lock | 67 +++++++++++++++---------------- pyproject.toml | 9 ++++- 4 files changed, 53 insertions(+), 37 deletions(-) diff --git a/.github/workflows/meltano-run.yml b/.github/workflows/meltano-run.yml index bd33e17..acde957 100644 --- a/.github/workflows/meltano-run.yml +++ b/.github/workflows/meltano-run.yml @@ -28,6 +28,11 @@ jobs: output_db: nested.db target_table: users install_extractor: false + - tap: tap-csv + mapping: fake + output_db: fake.db + target_table: customers + install_extractor: false steps: - uses: actions/checkout@v4 diff --git a/meltano.yml b/meltano.yml index 1e45ca9..fdacfb1 100644 --- a/meltano.yml +++ b/meltano.yml @@ -36,7 +36,7 @@ plugins: mappers: - name: meltano-map-transformer namespace: meltano_map_transformer - pip_url: -e . + pip_url: -e .[faker] executable: meltano-map-transform settings: - name: stream_maps @@ -72,6 +72,13 @@ plugins: users: id: id fields: "[f for f in fields if f['key'] != 'age']" + - name: fake + config: + stream_maps: + customers: + cc: fake.credit_card_number() # add a new field with a fake credit card number + stream_map_config: + faker_seed: 123456 environments: - name: dev config: diff --git a/poetry.lock b/poetry.lock index dc5dd8e..6e2eaef 100644 --- a/poetry.lock +++ b/poetry.lock @@ -376,6 +376,21 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "faker" +version = "22.6.0" +description = "Faker is a Python package that generates fake data for you." +optional = true +python-versions = ">=3.8" +files = [ + {file = "Faker-22.6.0-py3-none-any.whl", hash = "sha256:2b57f0256da6b45b7851dca87836ef5e2ae2fbb64d63d8697f1e47830d7b505d"}, + {file = "Faker-22.6.0.tar.gz", hash = "sha256:fa6d969728ef3da6229da91267a1bd4e6b902044c4822012d4fc46c71bb92b26"}, +] + +[package.dependencies] +python-dateutil = ">=2.4" +typing-extensions = {version = ">=3.10.0.1", markers = "python_version <= \"3.8\""} + [[package]] name = "fs" version = "2.4.16" @@ -479,13 +494,13 @@ files = [ [[package]] name = "importlib-metadata" -version = "6.11.0" +version = "7.0.1" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-6.11.0-py3-none-any.whl", hash = "sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b"}, - {file = "importlib_metadata-6.11.0.tar.gz", hash = "sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443"}, + {file = "importlib_metadata-7.0.1-py3-none-any.whl", hash = "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e"}, + {file = "importlib_metadata-7.0.1.tar.gz", hash = "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"}, ] [package.dependencies] @@ -599,16 +614,6 @@ files = [ importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} referencing = ">=0.31.0" -[[package]] -name = "memoization" -version = "0.4.0" -description = "A powerful caching library for Python, with TTL support and multiple algorithm options. (https://github.com/lonelyenvoy/python-memoization)" -optional = false -python-versions = ">=3, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" -files = [ - {file = "memoization-0.4.0.tar.gz", hash = "sha256:fde5e7cd060ef45b135e0310cfec17b2029dc472ccb5bbbbb42a503d4538a135"}, -] - [[package]] name = "packaging" version = "23.2" @@ -836,17 +841,6 @@ files = [ [package.extras] cli = ["click (>=5.0)"] -[[package]] -name = "pytz" -version = "2023.4" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -files = [ - {file = "pytz-2023.4-py2.py3-none-any.whl", hash = "sha256:f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a"}, - {file = "pytz-2023.4.tar.gz", hash = "sha256:31d4583c4ed539cd037956140d695e42c033a19e984bfce9964a3f7d59bc2b40"}, -] - [[package]] name = "pyyaml" version = "6.0.1" @@ -1187,13 +1181,13 @@ files = [ [[package]] name = "singer-sdk" -version = "0.34.1" +version = "0.35.0a5" description = "A framework for building Singer taps" optional = false -python-versions = ">=3.7.1" +python-versions = ">=3.8" files = [ - {file = "singer_sdk-0.34.1-py3-none-any.whl", hash = "sha256:2b9afa40722c2d7288992d86d16298128e38387941306267d908b9cd954227a3"}, - {file = "singer_sdk-0.34.1.tar.gz", hash = "sha256:5da11da3de07cc31cc8c1f3f19a40b8f8be4a0df9b5535fb90880c263e58aee5"}, + {file = "singer_sdk-0.35.0a5-py3-none-any.whl", hash = "sha256:220145bacb7709825acbb59d69faa324a9c26571b161fc7798dd7bafc81b8f1a"}, + {file = "singer_sdk-0.35.0a5.tar.gz", hash = "sha256:ed099380634668c61f3bd83693b493243fa26359912e82495409422e8c8b358b"}, ] [package.dependencies] @@ -1202,20 +1196,19 @@ backports-datetime-fromisoformat = {version = ">=2.0.1", markers = "python_versi click = ">=8.0,<9.0" cryptography = ">=3.4.6" fs = ">=2.4.16" -importlib-metadata = {version = "<7.0.0", markers = "python_version < \"3.12\""} +importlib-metadata = {version = "<8.0.0", markers = "python_version < \"3.12\""} importlib-resources = {version = ">=5.12.0", markers = "python_version < \"3.9\""} inflection = ">=0.5.1" joblib = ">=1.0.1" jsonpath-ng = ">=1.5.3" -jsonschema = {version = ">=4.16.0", markers = "python_version >= \"3.8\""} -memoization = {version = ">=0.3.2,<0.5.0", markers = "python_version < \"4\""} +jsonschema = ">=4.16.0" packaging = ">=23.1" -pendulum = {version = ">=2.1.0,<4", markers = "python_version >= \"3.8\""} +pendulum = ">=2.1.0,<4" PyJWT = ">=2.4,<3.0" python-dateutil = ">=2.8.2" python-dotenv = ">=0.20" -pytz = ">=2022.2.1" PyYAML = ">=6.0" +referencing = ">=0.30.0" requests = ">=2.25.1" simpleeval = ">=0.9.13" simplejson = ">=3.17.6" @@ -1225,7 +1218,8 @@ urllib3 = ">=1.26,<2" [package.extras] docs = ["furo (>=2022.12.7)", "myst-parser (>=1)", "sphinx (>=4.5)", "sphinx-autobuild (>=2021.3.14)", "sphinx-copybutton (>=0.3.1)", "sphinx-inline-tabs (>=2023.4.21)", "sphinx-notfound-page (>=1.0.0)", "sphinx-reredirects (>=0.1.1)"] -parquet = ["numpy (<1.22)", "numpy (>=1.22)", "numpy (>=1.22,<1.25)", "pyarrow (>=11,<13)", "pyarrow (>=13)"] +faker = ["faker (>=22.5,<23.0)"] +parquet = ["numpy (>=1.22)", "numpy (>=1.22,<1.25)", "pyarrow (>=13)"] s3 = ["fs-s3fs (>=1.1.1)"] testing = ["pytest (>=7.2.1)", "pytest-durations (>=1.2.0)"] @@ -1391,7 +1385,10 @@ files = [ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] +[extras] +faker = ["faker"] + [metadata] lock-version = "2.0" python-versions = ">=3.8" -content-hash = "2b7d9568791569ee5d5af07e78648f0555702bc4e0c5932a53499b53ead1cbf9" +content-hash = "e9dfae16725cee8a873d32b8d7a3b8cbcc7453bdc05316c6087ac198132e0a12" diff --git a/pyproject.toml b/pyproject.toml index 1dbc73f..ca9050d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,11 +18,18 @@ documentation = "https://github.com/MeltanoLabs/meltano-map-transform#readme" [tool.poetry.dependencies] python = ">=3.8" -singer-sdk = "~=0.34.0" +faker = { version = "~=22.6", optional = true } + +[tool.poetry.dependencies.singer-sdk] +version = ">=0.34.0" +allow-prereleases = true [tool.poetry.dev-dependencies] pytest = ">=7.4.4" +[tool.poetry.extras] +faker = ["faker"] + [tool.poetry-dynamic-versioning] enable = true format-jinja = """