diff --git a/data/extract/extractors.meltano.yml b/data/extract/extractors.meltano.yml index 681e780b..c927d94c 100644 --- a/data/extract/extractors.meltano.yml +++ b/data/extract/extractors.meltano.yml @@ -234,3 +234,10 @@ plugins: select: - '*meltano_activity_notifications.title' - '*meltano_activity_notifications.body' + - name: tap-smoke-test + variant: meltano + pip_url: git+https://github.com/meltano/tap-smoke-test.git + config: + streams: + - stream_name: animals + input_filename: https://raw.githubusercontent.com/meltano/tap-smoke-test/main/demo-data/animals-data.jsonl diff --git a/data/load/loaders.meltano.yml b/data/load/loaders.meltano.yml index 676caf87..ccdfb657 100644 --- a/data/load/loaders.meltano.yml +++ b/data/load/loaders.meltano.yml @@ -41,3 +41,6 @@ plugins: inherit_from: target-apprise - name: target-apprise-meltano-activity inherit_from: target-apprise + - name: target-jsonl + variant: andyh1203 + pip_url: target-jsonl \ No newline at end of file diff --git a/data/orchestrate/orchestrators.meltano.yml b/data/orchestrate/orchestrators.meltano.yml index 2e412bde..08c5af91 100644 --- a/data/orchestrate/orchestrators.meltano.yml +++ b/data/orchestrate/orchestrators.meltano.yml @@ -68,8 +68,8 @@ schedules: job: hubspot_publish - name: sample_schedule - interval: 0 12 * * * - job: sample_schedule + interval: 0 0 * * * + job: sample_job jobs: @@ -173,6 +173,6 @@ jobs: - dbt-snowflake:run_staging_cloudwatch - dbt-snowflake:test_staging_cloudwatch -- name: sample_schedule +- name: sample_job tasks: - - dbt-snowflake:describe + - tap-smoke-test target-jsonl diff --git a/data/plugins/extractors/tap-smoke-test--meltano.lock b/data/plugins/extractors/tap-smoke-test--meltano.lock new file mode 100644 index 00000000..a47c48c7 --- /dev/null +++ b/data/plugins/extractors/tap-smoke-test--meltano.lock @@ -0,0 +1,65 @@ +{ + "plugin_type": "extractors", + "name": "tap-smoke-test", + "namespace": "tap_smoke_test", + "variant": "meltano", + "label": "Smoke Test", + "docs": "https://hub.meltano.com/extractors/tap-smoke-test--meltano", + "repo": "https://github.com/meltano/tap-smoke-test", + "pip_url": "git+https://github.com/meltano/tap-smoke-test.git", + "executable": "tap-smoke-test", + "description": "Generates sample data to be used for testing.", + "logo_url": "https://hub.meltano.com/assets/logos/extractors/smoke-test.png", + "capabilities": [ + "about", + "catalog", + "discover", + "schema-flattening", + "state", + "stream-maps" + ], + "settings_group_validation": [ + [ + "streams" + ] + ], + "settings": [ + { + "name": "flattening_enabled", + "kind": "boolean", + "label": "Flattening Enabled", + "description": "'True' to enable schema flattening and automatically expand nested properties." + }, + { + "name": "flattening_max_depth", + "kind": "integer", + "label": "Flattening Max Depth", + "description": "The max depth to flatten schemas." + }, + { + "name": "schema_inference_record_count", + "kind": "integer", + "value": 5, + "label": "Schema Inference Record Count", + "description": "How many records of the source data should be used for schema inference/construction." + }, + { + "name": "stream_map_config", + "kind": "object", + "label": "Stream Map Config", + "description": "User-defined config values to be used within map expressions." + }, + { + "name": "stream_maps", + "kind": "object", + "label": "Stream Maps", + "description": "Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html)." + }, + { + "name": "streams", + "kind": "array", + "label": "Streams", + "description": "An array of objects containing:\n* `stream_name`: The name of the stream.\n* `input_filename`: Path to a jsonl file containing records to use for mock data.\n* `client_exception`: (Default False) Whether we should simulate failing by having the client raise an exception.\n* `schema_gen_exception`: (Default False) Whether we should simulate failing by raising an exception during schema inference.\n* `loop_count`: (Default 1) The number of times we should playback the input file.\n\nFor example:\n\n```yaml\nstreams:\n- stream_name: animals\n input_filename: https://github.com/meltano/tap-smoke-test/blob/main/demo-data/animals-data.jsonl\n```\n" + } + ] +} \ No newline at end of file diff --git a/data/plugins/loaders/target-jsonl--andyh1203.lock b/data/plugins/loaders/target-jsonl--andyh1203.lock new file mode 100644 index 00000000..5825fc4a --- /dev/null +++ b/data/plugins/loaders/target-jsonl--andyh1203.lock @@ -0,0 +1,34 @@ +{ + "plugin_type": "loaders", + "name": "target-jsonl", + "namespace": "target_jsonl", + "variant": "andyh1203", + "label": "JSON Lines (JSONL)", + "docs": "https://hub.meltano.com/loaders/target-jsonl--andyh1203", + "repo": "https://github.com/andyh1203/target-jsonl", + "pip_url": "target-jsonl", + "description": "JSONL loader", + "logo_url": "https://hub.meltano.com/assets/logos/loaders/jsonl.png", + "settings": [ + { + "name": "destination_path", + "kind": "string", + "value": "output", + "label": "Destination Path", + "description": "Sets the destination path the JSONL files are written to, relative\nto the project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write JSONL files to the project root, set an empty string (`\"\"`).\n" + }, + { + "name": "do_timestamp_file", + "kind": "boolean", + "value": false, + "label": "Include Timestamp in File Names", + "description": "Specifies if the files should get timestamped.\n\nBy default, the resulting file will not have a timestamp in the file name (i.e. `exchange_rate.jsonl`).\n\nIf this option gets set to `true`, the resulting file will have a timestamp associated with it (i.e. `exchange_rate-{timestamp}.jsonl`).\n" + }, + { + "name": "custom_name", + "kind": "string", + "label": "Custom File Name Override", + "description": "Specifies a custom name for the filename, instead of the stream name.\n\nThe file name will be `{custom_name}-{timestamp}.jsonl`, if `do_timestamp_file` is `true`.\nOtherwise the file name will be `{custom_name}.jsonl`.\n\nIf custom name is not provided, the stream name will be used.\n" + } + ] +} \ No newline at end of file