diff --git a/.github/labeler.yml b/.github/labeler.yml index 300c1c71..3bc66543 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,196 +2,205 @@ ## /api -'api': - - 'content/api/**' +"api": + - "content/api/**" -'api:community': - - 'content/api/community/**' +"api:community": + - "content/api/community/**" -'api:official': - - 'content/api/official/**' +"api:official": + - "content/api/official/**" ## /cli -'cli': - - 'content/cli/**' +"cli": + - "content/cli/**" -'cli:advanced': - - 'content/cli/advanced/**' +"cli:advanced": + - "content/cli/advanced/**" -'cli:client': - - 'content/cli/client/**' +"cli:client": + - "content/cli/client/**" -'cli:cloud': - - 'content/cli/cloud/**' +"cli:cloud": + - "content/cli/cloud/**" -'cli:commands': - - 'content/cli/commands/**' +"cli:commands": + - "content/cli/commands/**" -'cli:local': - - 'content/cli/local/**' +"cli:local": + - "content/cli/local/**" -'cli:smartmodules': - - 'content/cli/smartmodules/**' +"cli:smartmodules": + - "content/cli/smartmodules/**" -'cli:utilities': - - 'content/cli/utilities/**' +"cli:connectors": + - "content/cli/connectors/**" + +"cli:utilities": + - "content/cli/utilities/**" ## /connectors -'connectors': - - 'content/connectors/**' +"connectors": + - "content/connectors/**" + +"connectors:examples": + - "content/connectors/examples/**" + +"connectors:how-to": + - "content/connectors/how-to/**" -'connectors:examples': - - 'content/connectors/examples/**' +"connectors:images": + - "content/connectors/images/**" -'connectors:how-to': - - 'content/connectors/how-to/**' +"connectors:inbound": + - "content/connectors/inbound/**" -'connectors:images': - - 'content/connectors/images/**' +"connectors:outbound": + - "content/connectors/outbound/**" -'connectors:inbound': - - 'content/connectors/inbound/**' +"connectors:cloud-connectors": + - "content/connectors/cloud-connectors.md" -'connectors:outbound': - - 'content/connectors/outbound/**' +"connectors:connector-template": + - "content/connectors/connector-template.md" -'connectors:cloud-connectors': - - 'content/connectors/cloud-connectors.md' +"connectors:local-connectors": + - "content/connectors/local-connectors.md" -'connectors:connector-template': - - 'content/connectors/connector-template.md' +"connectors-beta": + - "content/connectors-beta/**" -'connectors:local-connectors': - - 'content/connectors/local-connectors.md' +"cdk": + - "content/connectors-beta/cdk/**" ## /docs -'docs': - - 'content/docs/**' +"docs": + - "content/docs/**" -'docs:architecture': - - 'content/docs/architecture/**' +"docs:architecture": + - "content/docs/architecture/**" -'docs:clients': - - 'content/docs/clients/**' +"docs:clients": + - "content/docs/clients/**" -'docs:concepts': - - 'content/docs/concepts/**' +"docs:concepts": + - "content/docs/concepts/**" -'docs:general': - - 'content/docs/general/**' +"docs:general": + - "content/docs/general/**" -'docs:get-started': - - 'content/docs/get-started/**' +"docs:get-started": + - "content/docs/get-started/**" -'docs:interfaces': - - 'content/docs/interfaces/**' +"docs:interfaces": + - "content/docs/interfaces/**" -'docs:kubernetes': - - 'content/docs/kubernetes/**' +"docs:kubernetes": + - "content/docs/kubernetes/**" -'docs:operations': - - 'content/docs/operations/**' +"docs:operations": + - "content/docs/operations/**" -'docs:tutorials': - - 'content/docs/tutorials/**' +"docs:tutorials": + - "content/docs/tutorials/**" -'docs:core-capabilities': - - 'content/docs/core-capabilities.md' +"docs:core-capabilities": + - "content/docs/core-capabilities.md" -'docs:key-features': - - 'content/docs/key-features.md' +"docs:key-features": + - "content/docs/key-features.md" -'docs:samples': - - 'content/docs/samples.md' +"docs:samples": + - "content/docs/samples.md" ## /download -'download': - - 'content/download/**' +"download": + - "content/download/**" ## /news -'this-week-in-fluvio': - - 'content/news/**' - - 'data/twif/**' +"this-week-in-fluvio": + - "content/news/**" + - "data/twif/**" ## /smartmodules -'smartmodules': - - 'content/smartmodules/**' +"smartmodules": + - "content/smartmodules/**" -'smartmodules:analytics': - - 'content/smartmodules/analytics/**' +"smartmodules:analytics": + - "content/smartmodules/analytics/**" -'smartmodules:certified': - - 'content/smartmodules/certified/**' +"smartmodules:certified": + - "content/smartmodules/certified/**" -'smartmodules:certified:jolt': - - 'content/smartmodules/certified/jolt.md' +"smartmodules:certified:jolt": + - "content/smartmodules/certified/jolt.md" -'smartmodules:certified:json-sql': - - 'content/smartmodules/certified/json-sql.md' +"smartmodules:certified:json-sql": + - "content/smartmodules/certified/json-sql.md" -'smartmodules:hub': - - 'content/smartmodules/hub/**' +"smartmodules:hub": + - "content/smartmodules/hub/**" -'smartmodules:images': - - 'content/smartmodules/images/**' +"smartmodules:images": + - "content/smartmodules/images/**" -'smartmodules:smdk': - - 'content/smartmodules/smdk/**' +"smartmodules:smdk": + - "content/smartmodules/smdk/**" -'smartmodules:transform': - - 'content/smartmodules//**' +"smartmodules:transform": + - "content/smartmodules//**" # data for templates or automation -'data-content': - - 'content/data/**' +"data-content": + - "content/data/**" # embeddable content -'embedded-content': - - 'content/docs/**' +"embedded-content": + - "content/docs/**" # functions -'functions': - - 'content/functions/**' +"functions": + - "content/functions/**" -'functions:newsletter': - - 'content/functions/newsletter/**' +"functions:newsletter": + - "content/functions/newsletter/**" # Below is for Hugo or editor related -'hugo': - - 'config.toml' +"hugo": + - "config.toml" # layouts -'hugo:layouts': - - 'content/layouts/**' +"hugo:layouts": + - "content/layouts/**" -'partials': - - 'content/layouts/partials/**' +"partials": + - "content/layouts/partials/**" -'shortcodes': - - 'content/layouts/shortcodes**' +"shortcodes": + - "content/layouts/shortcodes**" # scripts -'hugo:scripts': - - 'content/scripts/**' - - 'hugo-start.sh' +"hugo:scripts": + - "content/scripts/**" + - "hugo-start.sh" # static content -'hugo:static': - - 'content/static/**' +"hugo:static": + - "content/static/**" # editor reference -'editor:reference': - - 'WRITERS.md' - - 'README.md' - - '*.excalidraw' +"editor:reference": + - "WRITERS.md" + - "README.md" + - "*.excalidraw" # CI -'CI': - - '.github/**' - - 'bors.toml' - - 'netlify.toml' - - '.htmltest.yml' - - 'Makefile' +"CI": + - ".github/**" + - "bors.toml" + - "netlify.toml" + - ".htmltest.yml" + - "Makefile" diff --git a/.github/workflows/check_examples.yml b/.github/workflows/check_examples.yml index bfd387b5..13ab35ee 100644 --- a/.github/workflows/check_examples.yml +++ b/.github/workflows/check_examples.yml @@ -77,7 +77,7 @@ jobs: - name: Build and test clients if: ${{ needs.test-skip-check.outputs.no-skip }} - uses: nick-fields/retry@v2.8.2 + uses: nick-fields/retry@v2.8.3 with: timeout_minutes: 10 max_attempts: 3 diff --git a/.github/workflows/htmltest.yml b/.github/workflows/htmltest.yml index f5c6c1f5..55a587d3 100644 --- a/.github/workflows/htmltest.yml +++ b/.github/workflows/htmltest.yml @@ -47,8 +47,9 @@ jobs: - name: Run Hugo in the background if: ${{ needs.test-skip-check.outputs.no-skip }} run: ./hugo-start.sh & + # Skip running for now - name: Run htmltest - if: ${{ needs.test-skip-check.outputs.no-skip }} + if: ${{ false }} run: | make htmltest diff --git a/.gitignore b/.gitignore index ea7924f8..61c09b0e 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ bin/ # Rust embeds/client-examples/rust/target/ +target/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..187631a8 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,569 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d7ae14b20b94cb02149ed21a86c423859cbe18dc7ed69845cace50e52b40a5" +dependencies = [ + "bitflags", + "clap_derive", + "clap_lex", + "is-terminal", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44bec8e5c9d09e439c4335b1af0abaab56dcf3b94999a936e1bb47b9134288f0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350b9cf31731f9957399229e9b2adc51eeabdfbe9d71d9a0552275fd12710d09" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fluvio-cms" +version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "convert_case", + "lazy_static", + "strum", + "tracing", + "tracing-subscriber", + "xshell", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "io-lifetimes" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfa919a82ea574332e2de6e74b4c36e74d41982b335080fa59d4ef31be20fdf3" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "is-terminal" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" +dependencies = [ + "hermit-abi", + "io-lifetimes", + "rustix", + "windows-sys", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" + +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "os_str_bytes" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" + +[[package]] +name = "rustix" +version = "0.36.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "xshell" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962c039b3a7b16cf4e9a4248397c6585c07547412e7d6a6e035389a802dcfe90" +dependencies = [ + "xshell-macros", +] + +[[package]] +name = "xshell-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dbabb1cbd15a1d6d12d9ed6b35cc6777d4af87ab3ba155ea37215f20beab80c" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..abb87bba --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,4 @@ +[workspace] +members = [ + "scripts/fluvio-cms" +] diff --git a/Makefile b/Makefile index 2f2a4c1c..815648f4 100644 --- a/Makefile +++ b/Makefile @@ -43,3 +43,5 @@ run-client-example-java: run-client-example run-client-example-python: LANG=python run-client-example-python: run-client-example +update-cli-help: + ./scripts/update-cli-help.sh \ No newline at end of file diff --git a/README.md b/README.md index 62f27394..c2107412 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,49 @@ Hugo watches for file changes and automatically updates website. * `stable` updates [fluvio.io](https://fluvio.io) * `master` updates [nightly.fluvio.io](https://nightly.fluvio.io) + +## Connector-beta reference docs + +A preview of the `fluvio-cms` CLI can be used to update the reference docs for connectors in the Hub. + +Connectors are organized in`scripts/fluvio-cms/src/connectors/mod.rs`, and define: +- Protocol used by connector +- Direction of the data flow with respect to a Fluvio cluster. +- Location in Hugo repo where connector README is stored +- Location in Hugo repo where the content template is stored + +At the time of this writing, reference docs are downloaded from either the public github repo, or from a local clone, but this will transition to the Connector package data. + +From public repo: +- Inbound HTTP +- Inbound MQTT + +From local disk: +- Inbound Kafka (*) +- Outbound Kafka (*) +- Outbound SQL + +(*) Kafka docs were manually separated from a single file into multiple + +### Add new connector docs + +2 areas need to be updated before `fluvio-cms` will support a new connector's docs + +1. For new protocols, add to the `DataService` struct. +2. Depending on the connector's data directionality, add an entry to the `INBOUND` or `OUTBOUND` hashmap + +### Update existing connector docs + +Downloading from the public git repo: + +Ex. Download MQTT docs from github repo +```shell +$ cargo run -- connector --protocol mqtt --direction inbound +``` + +Using README from local git clone: + +Ex. Copy SQL docs +```shell +$ cargo run -- connector --protocol sql --direction outbound --file /path/to/README +``` \ No newline at end of file diff --git a/config.toml b/config.toml index bbd50835..c99292ed 100644 --- a/config.toml +++ b/config.toml @@ -43,37 +43,37 @@ staticDir = ["static", "code"] identifier = "connectors" name = "Connectors" url = "/connectors/" - weight = 101 + weight = 120 [[menu.fluvio]] identifier = "smartmodules" name = "SmartModules" url = "/smartmodules/" - weight = 102 + weight = 130 [[menu.fluvio]] identifier = "cli" name = "CLI" url = "/cli/" - weight = 103 + weight = 140 [[menu.fluvio]] identifier = "api" name = "APIs" url = "/api/" - weight = 104 + weight = 150 [[menu.fluvio]] identifier = "news" name = "News" url = "/news/" - weight = 105 + weight = 160 [[menu.fluvio]] identifier = "chat" url = "https://discordapp.com/invite/bBG2dTz" pre = "fab fa-discord" - weight = 160 + weight = 170 [[menu.infinyon]] identifier = "blog" @@ -95,3 +95,6 @@ staticDir = ["static", "code"] url = "https://www.infinyon.com/use-cases" pre = "fas fa-tools" weight = 120 + +[permalinks] + connectors-old = '/deprecated/connectors-v2/:sections[1:]/:slug' \ No newline at end of file diff --git a/content/api/official/python/examples.md b/content/api/official/python/examples.md index e3315448..a7005b97 100644 --- a/content/api/official/python/examples.md +++ b/content/api/official/python/examples.md @@ -1,17 +1,9 @@ --- title: Python Examples -menu: Examples +menu: Examples weight: 20 --- -{{< caution >}} - -Only producers and consumers are supported. There is no support for cluster administration. - -Also client does not support [async python](https://docs.python.org/3/library/asyncio.html). - -{{}} - * The Python client [wraps the rust client](https://www.infinyon.com/blog/2021/03/python-client/). * It currently does not support the administrator features that the rust client does. * The [PartitionConsumer.stream](https://infinyon.github.io/fluvio-client-python/fluvio.html#PartitionConsumer.stream) returns an object which implements the [python iterator convention](https://wiki.python.org/moin/Iterator) to allow for iterating over the stream in a for-loop. @@ -21,15 +13,50 @@ To see the full docs, visit our [pdoc page](https://infinyon.github.io/fluvio-cl Follow the [installation instructions]({{< ref "installation.md" >}}) to run this example. -{{}} +### Prerequisites + +Create the topic used to produce and consume records: + +%copy% +```bash +fluvio topic create python-data +``` + +### Producer + +Create a file called `python-produce.py`: + +{{}} + +Let's run the file: -### Run +%copy first-line% +```shell +$ python python-produce.py +``` + +### Consumer + +Create a file called `python-consume.py`: + +{{}} + +Let's run the file: %copy first-line% ```shell -$ python hello-python.py +$ python python-consume.py ``` +## Limitations +* Fluvio cluster administration is not supported. +* Python [async](https://docs.python.org/3/library/asyncio.html) is not supported. + +## Example with a SmartModule + +{{}} + + ## Links to Docs: * [Connect to Fluvio](https://infinyon.github.io/fluvio-client-python/fluvio.html#Fluvio.connect) * [Get a Producer](https://infinyon.github.io/fluvio-client-python/fluvio.html#Fluvio.topic_producer) diff --git a/content/cli/cloud/connector.md b/content/cli/cloud/connector.md index e055a3c4..834992af 100644 --- a/content/cli/cloud/connector.md +++ b/content/cli/cloud/connector.md @@ -13,6 +13,8 @@ $ fluvio cloud connector -h {{% inline-embed file="embeds/cli/help/fluvio-cloud-connector.md" %}} +-> For more info about using connectors, see the [Connectors page]({{}}). The available connector types are listed under the *Inbound* and *Outbound* sections. + --- ## `fluvio cloud connector create` @@ -28,7 +30,7 @@ $ fluvio cloud connector create -h To create a connector, you need to create a YAML-based connector config file. -For more about the connector config file, see the [Cloud connectors page]({{}}) or the [connector template]({{}}) +For more about the connector config file, see the [Cloud connectors page]({{}}). When running `fluvio cloud connector create`, pass the path to this file using the `--config` option. @@ -56,19 +58,7 @@ $ fluvio cloud connector config -h Example usage: -%copy first-line% -```bash -$ fluvio cloud connector config cat-facts - -name: cat-facts -type: http-source -topic: cat-facts -version: 0.4.2 -parameters: - endpoint: https://catfact.ninja/fact - interval: 10s - -``` +{{}} --- diff --git a/content/cli/connectors/_index.md b/content/cli/connectors/_index.md new file mode 100644 index 00000000..ebe9dd96 --- /dev/null +++ b/content/cli/connectors/_index.md @@ -0,0 +1,6 @@ +--- +title: CONNECTORS +weight: 40 +_build: + render: never +--- diff --git a/content/cli/connectors/cdk.md b/content/cli/connectors/cdk.md new file mode 100644 index 00000000..fb21bdf5 --- /dev/null +++ b/content/cli/connectors/cdk.md @@ -0,0 +1,27 @@ +--- +title: Connector Development Kit (CDK) +menu: CDK +weight: 10 +--- + +Check out the [Connector Development Kit]({{}}) page for more information. + +{{% inline-embed file="embeds/cli/help/cdk.md" %}} +## `cdk build` +{{% inline-embed file="embeds/cli/help/cdk-build.md" %}} +## `cdk generate` +{{% inline-embed file="embeds/cli/help/cdk-generate.md" %}} +## `cdk test` +{{% inline-embed file="embeds/cli/help/cdk-test.md" %}} +## `cdk deploy` +{{% inline-embed file="embeds/cli/help/cdk-deploy.md" %}} +## `cdk deploy start` +{{% inline-embed file="embeds/cli/help/cdk-deploy-start.md" %}} +## `cdk deploy shutdown` +{{% inline-embed file="embeds/cli/help/cdk-deploy-shutdown.md" %}} +## `cdk deploy list` +{{% inline-embed file="embeds/cli/help/cdk-deploy-list.md" %}} +## `cdk deploy log` +{{% inline-embed file="embeds/cli/help/cdk-deploy-log.md" %}} +## `cdk publish` +{{% inline-embed file="embeds/cli/help/cdk-publish.md" %}} \ No newline at end of file diff --git a/content/cli/connectors/hub.md b/content/cli/connectors/hub.md new file mode 100644 index 00000000..de2a042e --- /dev/null +++ b/content/cli/connectors/hub.md @@ -0,0 +1,70 @@ +--- +title: Connectors Hub +menu: Hub +weight: 20 +--- + +## Introduction to the Connectors Hub Service + +The SmartConnectors Hub allows users to start building their data streaming pipeline with SmartConnectors which have been uploaded into the Hub as ready to use packages. The Hub allows Fluvio pipeline construction focusing on the business logic by providing already built, reliable and tested modules. The modules contain high level configurable inputs which can be applied without needing to rebuild the module. Pipelines can employ a mix of Hub downloaded packages as well as custom developed SmartModules. + +A view of available and installed SmartModules is accessible from both the `fluvio` cli as well as on the Fluvio cloud dashboard. + + +## Fluvio Hub Commands + +Commands for users to interact with the SmartModule hub are under the **fluvio hub connector** set of commands. + +- `fluvio hub connector download group/package@version` +- `fluvio hub connector list` + +### `fluvio hub connector download` + +{{% inline-embed file="embeds/cli/help/fluvio-hub-connector-download.md" %}} + +The command **fluvio hub connector download ** will download a hub SmartConnector `.ipkg` locally, which can be used with `cdk`. + +%copy first-line% +```bash +$ % fluvio hub connector download infinyon/mqtt-source@0.1.2 +downloading infinyon/mqtt-source@0.1.2 to infinyon-mqtt-source-0.1.2.ipkg +... downloading complete +``` + +### `fluvio hub connector list` + +The command **fluvio hub connector list** will list Connectors and versions available for download + +{{% inline-embed file="embeds/cli/help/fluvio-hub-connector-list.md" %}} + +The list command shows Connectors by **group/package@version** in the CONNECTOR column. The Visibility column describes if the package is accessible to public or private views. If package visibility is **private** only your user login can list or download the package. Packages uploaded by InfinyOn will be in the infinyon group. + + +%copy first-line% +```bash +$ fluvio hub connector list +Using hub https://hub.infinyon.cloud + CONNECTOR Visibility + infinyon/http-source@0.1.1 public + infinyon/kafka-sink@0.1.1 public + infinyon/kafka-source@0.4.0 public + infinyon/mqtt-source@0.1.2 public + infinyon/sql-sink@0.1.1 public +``` + +## Connector Packages + +Connector Packages are listed in the hub as in the form of `/@version`. + +Group `infinyon` packages are published by InfinyOn, the maker of Fluvio. Third-party developers may also upload public packages which appear in other groups. The Hub service enforces signing and publishing restrictions on uploaded packages. Once a signed package of given group has been published, only the group owner may update that package or add new packages within that group. + +### Private and Public Packages + +Published packages are by default private. In order to make a published package publicly available, the package should be set to public. The cli method to do this is below: + +```bash +cd connector-dev-dir +cdk set-public +``` + +After the package is set to public, any publish will ask to verify the publish operation. Once a package version is publicly published, the package cannot revoked. \ No newline at end of file diff --git a/content/cli/smartmodules/hub.md b/content/cli/smartmodules/hub.md index 8f2cc6c5..342dd7d9 100644 --- a/content/cli/smartmodules/hub.md +++ b/content/cli/smartmodules/hub.md @@ -6,9 +6,9 @@ weight: 20 ## Introduction to the SmartModule Hub Service -The SmartModule Hub allows users to start building their data streaming pipline with SmartModules which have been uploaded into the Hub as ready to use packages. The Hub allows Fluvio pipeline construction focusing on the business logic by providing already built, reliable and tested modules. The modules contain high level configurable inputs which can be applied without needing to rebuild the module. Pipelines can employ a mix of Hub downloaded packages as well as custom developed SmartModules. +The SmartModule Hub allows users to start building their data streaming pipeline with SmartModules which have been uploaded into the Hub as ready to use packages. The Hub allows Fluvio pipeline construction focusing on the business logic by providing already built, reliable and tested modules. The modules contain high level configurable inputs which can be applied without needing to rebuild the module. Pipelines can employ a mix of Hub downloaded packages as well as custom developed SmartModules. -A view of available and installed SmartModules is accessible from both the fluvio cli as well as on the Fluvio cloud dashboard. +A view of available and installed SmartModules is accessible from both the `fluvio` cli as well as on the Fluvio cloud dashboard. ## Fluvio Hub Commands @@ -79,14 +79,14 @@ Group `infinyon` packages are published by InfinyOn, the maker of Fluvio. Third ### Private and Public Packages -Published packages are by default private. In order to make a published package publically available, the package should be set to public. The cli method to do this is below: +Published packages are by default private. In order to make a published package publicly available, the package should be set to public. The cli method to do this is below: ```bash cd smartmodule-dev-dir smdk set-public ``` -After the package is set to public, any publish will ask to verify the publish operation. Once a package version is publically published, the package cannot revoked. +After the package is set to public, any publish will ask to verify the publish operation. Once a package version is publicly published, the package cannot revoked. diff --git a/content/connectors-old/_index.md b/content/connectors-old/_index.md new file mode 100644 index 00000000..1b54f94c --- /dev/null +++ b/content/connectors-old/_index.md @@ -0,0 +1,51 @@ +--- +title: Smart Connectors +menu: Overview +section: Connectors +toc: true +url: /deprecated/connectors-v2/ +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +Smart Connectors make the process of importing or exporting data simple. +You can import data with an `Inbound` connector and export data with an `Outbound` connector. + +Inbound and outbound connectors fundamentally work in the same way. The only difference is the direction your data is streaming with respect to a Fluvio topic. + +There are 4 steps to the connector: + +Smart Connectors + +- **Protocol**: Parses data according to the wire format of the connected data platform. +- **Extract**: Extracts raw data from the protocol format and packages it neatly into data structures + that may be used by subsequent stages or be produced directly to a topic. +- **Filter** (optional): A user-provided SmartModule that may determine whether a given record + should be discarded before sending it over the network to Fluvio, saving bandwidth. +- **Shape** (optional): A user-provided SmartModule that may take the extracted data structures and + transform them in to an application-specific format. + +The **Protocol** and **Extract** stages are built directly into the +connector. They offer basic access to your data through the various protocols your data sources use. + +In the **Extract** stage, your data is structured from whatever protocol it is sourced from. + +Additionally, You can apply custom pre-processing or post-processing to data, before it +arrives to or while it streams from a Fluvio topic. The **Filter** and **Shape** stages are provided through SmartModules. + +Powered by WebAssembly (also called wasm), SmartModules are pre-packaged or user-provided operations such as filters, maps, or aggregators that can be applied to records at various points in the streaming pipeline. +Supporting access to your data while it is in transit provides you the ability to clean, transform and enrich your data before it is stored in a topic, or it exits the Fluvio cluster. + +Use Connectors either as: +* a [Local Connector]({{}}) + * Run your connector on your machine as a docker container + +* a [Cloud Connector]({{}}), + * You can start a connector on [InfinyOn Cloud], and let us manage the infrastruture + +You can customize how your connectors run through a configuration file. + +For more info about connectors or configuration, check out our supported Inbound and Outbound connector docs. + +[InfinyOn Cloud]: https://infinyon.cloud diff --git a/content/connectors-old/cloud-connectors.md b/content/connectors-old/cloud-connectors.md new file mode 100644 index 00000000..6da1417a --- /dev/null +++ b/content/connectors-old/cloud-connectors.md @@ -0,0 +1,127 @@ +--- +title: Cloud Connectors +weight: 30 +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +Connectors with [InfinyOn Cloud] is the best option for those who want to manage their data pipelines in one place. + +Configuring connectors works the same way with InfinyOn Cloud, as it does locally. + +You can create and maintain your connectors in a more streamlined way through the CLI with [`fluvio cloud`]({{}}). + +## Create your first connector on InfinyOn Cloud + +This guide will walk you through creating an Inbound HTTP connector to ingest json data from and HTTP endpoint. + +To follow this guide you will need to sign up for [InfinyOn Cloud] and log into the CLI. + +%copy% +```bash +fluvio cloud login +``` + +{{}} +Check out the [`fluvio cloud` CLI docs]({{}}) for more details about logging into the CLI. +{{}} + +### Example HTTP connector +This is the config file for the [Inbound HTTP connector]({{}}) in this guide. + +{{}} + +In this config, we are creating a connector named `cat-facts`. It will request data from {{}} once every 30 seconds and receive json data. The connector will store the json into a topic called `cat-facts-data` + + +#### Start a connector + +You can create a connector by using `fluvio cloud connector create` with the example connector. + +%copy first-line% +```bash +$ fluvio cloud connector create --config catfacts-basic-connector.yml +connector "cat-facts" (http-source) created +``` +#### List all connectors + +After the connector is created, you can list the connectors you've created, and view their current status. + +%copy first-line% +```bash +$ fluvio cloud connector list + NAME TYPE VERSION STATUS + cat-facts http-source 0.3.0 Running +``` + +#### Look at connector logs + +If there is a need to debug the behavior of a connector, the logs are available by running `fluvio cloud connector logs cat-facts` + +%copy first-line% +```bash +$ fluvio cloud connector logs cat-facts +2022-10-23T07:36:36.915928Z INFO http_source: Starting HTTP source connector connector_version="0.3.0" git_hash="10ee08a94b7be7d91a31a01104b7f6e86e54b7d9" +2022-10-23T07:36:36.915981Z INFO http_source: interval=30s method=GET topic=cat-facts output_parts=body output_type=text endpoint=https://catfact.ninja/fact +2022-10-23T07:36:36.916165Z INFO fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c" +2022-10-23T07:36:36.920362Z INFO fluvio::fluvio: Connecting to Fluvio cluster fluvio_crate_version="0.12.14" fluvio_git_hash="" +2022-10-23T07:36:36.979270Z INFO connect: fluvio::sockets: connect to socket add=fluvio-sc-public:9003 +2022-10-23T07:36:37.025300Z INFO connect:connect_with_config: fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c" +2022-10-23T07:36:37.088073Z INFO connect:connect_with_config:connect: fluvio::sockets: connect to socket add=fluvio-sc-public:9003 +2022-10-23T07:36:37.494092Z INFO dispatcher_loop{self=MultiplexDisp(13)}: fluvio_socket::multiplexing: multiplexer terminated +2022-10-23T07:36:37.544828Z INFO http_source: Connected to Fluvio +2022-10-23T07:36:38.060832Z INFO run:create_serial_socket_from_leader{leader_id=0}:connect_to_leader{leader=0}:connect: fluvio::sockets: connect to socket add=fluvio-spu-main-0.acct-.svc.cluster.local:9005 +``` + +#### View data in topic + +The HTTP connector should be receiving data and storing it in a topic with the name we specified. + +%copy first-line% +```shell +$ fluvio topic list + NAME TYPE PARTITIONS REPLICAS RETENTION TIME COMPRESSION STATUS REASON + cat-facts-data computed 1 1 7days any resolution::provisioned +``` + +To verify, you can consume from the topic with the `fluvio consume` CLI. + +We are using the `-B` option to start from the beginning offset of the topic. Once you reach the end of the topic, you can see new data as it is sent to the topic. To exit this live view, press `Ctrl+C`. + +{{}} +Using the `--disable-continuous` flag with `fluvio consume` will exit the stream once the last record has printed to screen +{{}} + +```shell +$ fluvio consume cat-facts-data -B +{"fact":"Female felines are \\superfecund","length":31} +{"fact":"Cats only sweat through their paws and nowhere else on their body","length":65} +{"fact":"While many parts of Europe and North America consider the black cat a sign of bad luck, in Britain and Australia, black cats are considered lucky.","length":146} +^C +``` + +#### Delete a connector + +When you want to stop the connector, you can delete it with `fluvio cloud connector cat-facts` + +%copy first-line% +```shell +$ fluvio cloud connector delete cat-facts +connector "cat-facts" deleted +``` + +Deleting your connector will not delete the topic used by the connector. If you want to delete the topic, you can run `fluvio topic delete cat-facts-data` + +%copy first-line% +```shell +$ fluvio topic delete cat-facts-data +topic "cat-facts-data" deleted +``` + +### Conclusion + +We created a basic Inbound HTTP connector, looked at the logs for the connector, and viewed the HTTP response data in the Fluvio topic. Lastly, we deleted the connector and topic. + +You are ready to create your own connectors! Check out the docs for our supported Inbound and Outbound connectors to get started with your own data sources. + + +[InfinyOn Cloud]: https://infinyon.cloud diff --git a/content/connectors/connector-templates.md b/content/connectors-old/connector-templates.md similarity index 96% rename from content/connectors/connector-templates.md rename to content/connectors-old/connector-templates.md index d760df6a..f2e5e555 100644 --- a/content/connectors/connector-templates.md +++ b/content/connectors-old/connector-templates.md @@ -2,6 +2,7 @@ title: Connector template weight: 40 --- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} ## Connector Template diff --git a/content/connectors/examples/_index.md b/content/connectors-old/examples/_index.md similarity index 100% rename from content/connectors/examples/_index.md rename to content/connectors-old/examples/_index.md diff --git a/content/connectors/examples/github.md b/content/connectors-old/examples/github.md similarity index 99% rename from content/connectors/examples/github.md rename to content/connectors-old/examples/github.md index 7bb6c7b3..6885ae33 100644 --- a/content/connectors/examples/github.md +++ b/content/connectors-old/examples/github.md @@ -45,7 +45,7 @@ the file: %copy% ```yaml # connect.yml -version: 0.4.2 +version: 0.4.3 name: github-repo type: http-source topic: github-repo diff --git a/content/connectors/examples/java-connector.md b/content/connectors-old/examples/java-connector.md similarity index 94% rename from content/connectors/examples/java-connector.md rename to content/connectors-old/examples/java-connector.md index f3c139f8..ab09ca0f 100644 --- a/content/connectors/examples/java-connector.md +++ b/content/connectors-old/examples/java-connector.md @@ -8,7 +8,7 @@ This example `Dockerfile` packages our [example Java Fluvio project]({{}} -This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) +This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) ### docker-compose.yml diff --git a/content/connectors/examples/node-connector.md b/content/connectors-old/examples/node-connector.md similarity index 95% rename from content/connectors/examples/node-connector.md rename to content/connectors-old/examples/node-connector.md index d414fbf5..0cffbc76 100644 --- a/content/connectors/examples/node-connector.md +++ b/content/connectors-old/examples/node-connector.md @@ -8,7 +8,7 @@ This example `Dockerfile` packages our [example Node Fluvio project]({{}} -This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) +This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) ### docker-compose.yml diff --git a/content/connectors/examples/python-connector.md b/content/connectors-old/examples/python-connector.md similarity index 94% rename from content/connectors/examples/python-connector.md rename to content/connectors-old/examples/python-connector.md index 79022178..385d94ce 100644 --- a/content/connectors/examples/python-connector.md +++ b/content/connectors-old/examples/python-connector.md @@ -8,7 +8,7 @@ This example `Dockerfile` packages our [example Python Fluvio project]({{}} -This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) +This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) ### docker-compose.yml diff --git a/content/connectors/examples/rust-connector.md b/content/connectors-old/examples/rust-connector.md similarity index 94% rename from content/connectors/examples/rust-connector.md rename to content/connectors-old/examples/rust-connector.md index cc239f3a..9c3d5b23 100644 --- a/content/connectors/examples/rust-connector.md +++ b/content/connectors-old/examples/rust-connector.md @@ -8,7 +8,7 @@ This example `Dockerfile` packages [example Rust Fluvio project]({{}} -This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) +This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) ### docker-compose.yml diff --git a/content/connectors/how-to/_index.md b/content/connectors-old/how-to/_index.md similarity index 100% rename from content/connectors/how-to/_index.md rename to content/connectors-old/how-to/_index.md diff --git a/content/connectors/how-to/connectors-with-smartmodules.md b/content/connectors-old/how-to/connectors-with-smartmodules.md similarity index 99% rename from content/connectors/how-to/connectors-with-smartmodules.md rename to content/connectors-old/how-to/connectors-with-smartmodules.md index 17177b96..651abdb5 100644 --- a/content/connectors/how-to/connectors-with-smartmodules.md +++ b/content/connectors-old/how-to/connectors-with-smartmodules.md @@ -116,7 +116,7 @@ For this example, we would add `map` to the `parameters` section, like so: %copy% ```yaml # connect.yml -version: 0.4.2 +version: 0.4.3 name: cat-facts type: http-source topic: cat-facts diff --git a/content/connectors/how-to/development-workflow.md b/content/connectors-old/how-to/development-workflow.md similarity index 100% rename from content/connectors/how-to/development-workflow.md rename to content/connectors-old/how-to/development-workflow.md diff --git a/content/connectors/how-to/how-to-local-pgql-cloud.md b/content/connectors-old/how-to/how-to-local-pgql-cloud.md similarity index 100% rename from content/connectors/how-to/how-to-local-pgql-cloud.md rename to content/connectors-old/how-to/how-to-local-pgql-cloud.md diff --git a/content/connectors/how-to/how-to-pgsql-with-minikube.md b/content/connectors-old/how-to/how-to-pgsql-with-minikube.md similarity index 100% rename from content/connectors/how-to/how-to-pgsql-with-minikube.md rename to content/connectors-old/how-to/how-to-pgsql-with-minikube.md diff --git a/content/connectors/how-to/overview.md b/content/connectors-old/how-to/overview.md similarity index 100% rename from content/connectors/how-to/overview.md rename to content/connectors-old/how-to/overview.md diff --git a/content/connectors/how-to/python-connector.md b/content/connectors-old/how-to/python-connector.md similarity index 96% rename from content/connectors/how-to/python-connector.md rename to content/connectors-old/how-to/python-connector.md index 4d3c684d..668bbc73 100644 --- a/content/connectors/how-to/python-connector.md +++ b/content/connectors-old/how-to/python-connector.md @@ -35,7 +35,7 @@ This client is written in [Python], but we have client libraries for [Rust], [Ja [Java]: {{}} [Go]: {{}} -{{}} +{{}} Before we run this code, we need to create the fluvio topic that our client produces data to @@ -84,7 +84,7 @@ Then we create a new user `fluvio` with a home directory (in `/home/fluvio`). This is **required** for all connectors. The Fluvio cluster shares information with the `fluvio` user on startup. -{{}} +{{}} ### Build and Test the Container You can build the Docker image with this command. diff --git a/content/connectors/how-to/switching-profiles.md b/content/connectors-old/how-to/switching-profiles.md similarity index 100% rename from content/connectors/how-to/switching-profiles.md rename to content/connectors-old/how-to/switching-profiles.md diff --git a/content/connectors-old/inbound/_index.md b/content/connectors-old/inbound/_index.md new file mode 100644 index 00000000..b16ef9d6 --- /dev/null +++ b/content/connectors-old/inbound/_index.md @@ -0,0 +1,6 @@ +--- +title: INBOUND +weight: 10 +_build: + render: never +--- diff --git a/content/connectors-old/inbound/http.md b/content/connectors-old/inbound/http.md new file mode 100644 index 00000000..9cd1b9e5 --- /dev/null +++ b/content/connectors-old/inbound/http.md @@ -0,0 +1,221 @@ +--- +title: Inbound HTTP Connector +menu: HTTP +connector: + name: "infinyon/fluvio-connect-http-source" + link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/http" +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +Fluvio's `http` connector allows you to periodically fetch data from an HTTP endpoint +and feed the HTTP response into a Fluvio topic. + +This is useful for monitoring APIs +continuously and building streaming applications that react to new or updated info. + + +Note that this connector is _not_ intended for streaming HTTP endpoints, it instead +periodically sends HTTP requests and collects the response as an event. + +## Common config values + +%copy% +```yaml +type: http-source +``` + +%copy% +```yaml +version: 0.4.3 +``` + +## Parameters + +The inbound HTTP connector supports the following configuration options: + +### `endpoint` +*required* + +The HTTP endpoint to send requests to + +### `body` +*optional* + +The body to use in the HTTP request to the endpoint + +### `user_agent` +default: `fluvio/http-source X.Y.Z` + +The HTTP User-Agent request header used in the HTTP request. + +### `method` +default: `GET` + +Choices: +- `GET` +- `PUT` +- `POST` +- `DELETE` + +### `interval` +default: `10s` + +A time period between sending requests to the endpoint. + +The time is formatted as an integer and a suffix. Supported suffixes: + +- `msec`, `ms` -- milliseconds +- `seconds`, `second`, `sec`, `s` +- `minutes`, `minute`, `min`, `m` +- `hours`, `hour`, `hr`, `h` +- `days`, `day`, `d` + +### `headers` +*optional* + +A list additional headers to include in the HTTP request. In the form `Key=Value` + +Example + +```yaml +headers: + - Keep-Alive=timeout=5,max=1000 + - Cache-Control=no-cache +``` + +### `output_parts` +default: `body` + +HTTP Response output Record parts + +Choices: +* `body` +* `full` + +### `output_type` +default: `text` + +HTTP Response output Record type + +Choices: +* `text` +* `json` + +#### Example connector config + +{{}} + +This example config for inbound HTTP connector config reads from the {{}} website, and stores the output in the `cat-facts` topic every `300` seconds + +## Data Events + +The HTTP connector generates one data event per HTTP response. The data depends on the endpoint you are accessing and can be formatted in multiple ways, as described below. + +### HTTP - Body + +By default, the HTTP connector produces the `body` of the HTTP response in JSON format: + +Example output: `output_parts: body` + +%copy first-line% +```bash +$ fluvio consume cat-facts -T -d +{"fact":"A cat almost never meows at another cat, mostly just humans. Cats typically will spit, purr, and hiss at other cats.","length":116} +{"fact":"In one stride, a cheetah can cover 23 to 26 feet (7 to 8 meters).","length":65} +{"fact":"Phoenician cargo ships are thought to have brought the first domesticated cats to Europe in about 900 BC.","length":105} +``` + + +### HTTP - Header/Body + +Use the `output_parts` parameter to produce full HTTP responses: + +Example output: `output_parts: full` + +%copy first-line% +```bash +$ fluvio consume cat-facts -T=1 -d +Consuming records starting 1 from the end of topic 'cat-facts' +HTTP/1.1 200 OK +server: nginx +date: Thu, 28 Apr 2022 14:05:43 GMT +content-type: application/json +transfer-encoding: chunked +connection: keep-alive +vary: Accept-Encoding +cache-control: no-cache, private +x-ratelimit-limit: 100 +x-ratelimit-remaining: 98 +access-control-allow-origin: * +set-cookie: XSRF-TOKEN=eyJpdiI6IkgvdngxWVlsRUpiaXN2WFJnQTZyU1E9PSIsInZhbHVlIjoiMlVhQThFZmxCLzFnbVBrYWVubjBqS29TUENydkNibFp1bS9CajJtQkZvdk8vWHoxNFV2YmlZbVZIeU5JRDltWHU1ekRFamtrc2Mya0JGdGtuZlFkTFlHcmIwUTFCWURBOTlJZ1dHcld0VjlJcVp3cW1LZ2Z4RU8vTVRLS0FRVWMiLCJtYWMiOiI1ZTAwZTEwODkzMjRiNzFmNWU5YzJmZDVlNTgzZmI5Y2VkZDY4ZDMyOGU2MjNmZjEwZmJkZjMzZmI0NTI1YjgwIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 16:05:43 GMT; path=/; samesite=lax +set-cookie: cat_facts_session=eyJpdiI6ImF4c0p2aWM4TkJnUXdnK2psRUlyVWc9PSIsInZhbHVlIjoiR0Rkc3NjTU45cnpUc09pdHNEWk5kOW9BRFNTWVVUWDI2bE9yWWM2TDh2OW5LS0Uwbks2cXluR2R0QnEyYmMreWJYMlorb09WNjVEMkZSRGF5NGhYUDF6WjBac2lwYTJ3dVhlV3o1bXpsMGNoZ1hvcXo0YTNRczNqM1pheVR6ZHAiLCJtYWMiOiJiYTc1ZjA3NWEwY2IzYzkyZmIxNzdiOGY1YjQxZGNlMzU2ZjdiMjYwYzhkZjI0ZWJhNjkxZTY4NWExMDg3NzgxIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 16:05:43 GMT; path=/; httponly; samesite=lax +x-frame-options: SAMEORIGIN +x-xss-protection: 1; mode=block +x-content-type-options: nosniff + +{"fact":"There are more than 500 million domestic cats in the world, with approximately 40 recognized breeds.","length":100} +``` + +{{}} +The `header` is text and `body` is JSON. +{{}} + +### HTTP - JSON + +Set the `output_type` parameter to `json` if you want to convert the full HTTP response into JSON format. + + The following is the example output from {{}} with the following inbound HTTP connector config: +* `output_parts: full` +* `output_type: json` + +%copy first-line% +```bash +$ fluvio consume cat-facts -T=1 -d | jq +{ + "status": { + "version": "HTTP/1.1", + "code": 200, + "string": "OK" + }, + "header": { + "server": "nginx", + "date": "Thu, 28 Apr 2022 15:53:48 GMT", + "vary": "Accept-Encoding", + "cache-control": "no-cache, private", + "x-ratelimit-limit": "100", + "content-type": "application/json", + "x-ratelimit-remaining": "97", + "connection": "keep-alive", + "set-cookie": [ + "XSRF-TOKEN=eyJpdiI6IlFnMHN0UzFpMlBEaDhjdjBaRzNoN1E9PSIsInZhbHVlIjoibElGczZPaUFBdDc4eDhiT3N5bkJxZkRVOFpVTjdJaXI0VFFGS1dkU1ZodDV5SGVHOEF6bkhVWjFjTEduMWpuRU50WkV2WURML0JyWWdqWnBtWFpyUXJRU1BBU2Q1cTA1YmdMaWp4TGViRjJnVzNaMFFFdDBaVDhPOHVlRTQxTXUiLCJtYWMiOiJmMGJmY2Q1YmJmYjAwNTQ4NzM0ZmU0MzUxMGEzYzNiMDQ4NGIyOTE1ZjJiY2NkZjFlOWQzZGZmMDBiMWMwMmUwIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 17:53:48 GMT; path=/; samesite=lax", + "cat_facts_session=eyJpdiI6IklxdW1obmdqSFFSZ2ZRTmZGRnFNbXc9PSIsInZhbHVlIjoiSHJEQUJvZVVQbmJDNm9WaG5rbzNtYzBKZVN6Q0UwSGpXQTBRRHNxWGtVeVQ4bGxjU0lSZlhxa3g4SW0yS0k3bXpqNTIrK3R5K1Z3Vms5ck80dkdQWnlJWHhqWGpIY1R3Z25MWTBqTHAwZFlpQURGZUNFb2xRdmMwajVMaXV1TWkiLCJtYWMiOiJiN2Q3Yjk5MWE3Y2Q4ZWRkODk4MjM3YmM2NWJhZDI1MmY3ZmU0MDlkOWIyYzM0Zjg3YmE1NjdhNGMwZjliNjcyIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 17:53:48 GMT; path=/; httponly; samesite=lax" + ], + "x-xss-protection": "1; mode=block", + "x-content-type-options": "nosniff", + "transfer-encoding": "chunked", + "access-control-allow-origin": "*", + "x-frame-options": "SAMEORIGIN" + }, + "body": "{\"fact\":\"Most cats had short hair until about 100 years ago, when it became fashionable to own cats and experiment with breeding.\",\"length\":120}" +} +``` + +-> **JSON Output** "body" is encoded in quotes (\") as JSON String within due to HTTP Response in this example containing application/json body itself. + +To convert only the body of the HTTP Response and ignore the header, set `output_parts` to `body` and the `output_type` keep as `json`: + +```json +{ + "body": "{\"fact\":\"Most cats had short hair until about 100 years ago, when it became fashionable to own cats and experiment with breeding.\",\"length\":120}" +} +``` + + +## Changelog + +| Version | Date | PR | Subject | +|:-------:|:----------:|:----------------------------------------------------------------:| ------------------------------------------------- | +| 0.3.0 | 2022-02-01 | [278](https://github.com/infinyon/fluvio-connectors/pull/278) |
  • Add interval times in human readable times
  • Support for producer and consumer top level config options
      • | +| 0.2.0 | 2022-02-01 | [141](https://github.com/infinyon/fluvio-connectors/pull/141) |
      • Feature json Response Type Record
      • Deprecate metadata output_format in favor of `output_parts` [ body (default)
      • Add Metadata `output_type` [ text (default) | json ] | full ]
          • | +| 0.1.1 | 2022-01-31 | [127](https://github.com/infinyon/fluvio-connectors/pull/127) |
          • Feature full Response Parts Record
          • Add Metadata `output_format` [ body (default) | full ]
              • | +| 0.1.0 | 2021-11-09 | - |
              • Initial version with text/body Response (default) Record
              | diff --git a/content/connectors-old/inbound/kafka.md b/content/connectors-old/inbound/kafka.md new file mode 100644 index 00000000..9f16af37 --- /dev/null +++ b/content/connectors-old/inbound/kafka.md @@ -0,0 +1,59 @@ +--- +title: Inbound Kafka Connector +menu: Kafka +connector: + name: "infinyon/fluvio-connect-kafka-source" + link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/kafka" +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +The inbound Kafka Connector will send every record on a Kafka topic/partition to a Fluvio topic/partition. + +{{}} +The Inbound Kafka connector does not currently support SSL +{{}} + +## Common config values + +%copy% +```yaml +type: kafka-source +``` + +%copy% +```yaml +version: 0.3.2 +``` + +## Parameters + +The inbound Kafka connector supports the following configuration options: + +### `kakfa-url` +*required* + +The url of the Kafka instance to connect to. + +### `kafka-topic` +Default: Same name as the configured `fluvio` topic + +The name of the Kafka topic name to connect to. + +### `kafka-partition` +Default: `0` + +The Kafka partition to connect to. + +### `kafka-group` +Default: `fluvio-kafka-source` + +The Kafka group + +#### Example connector config + +{{}} + +## Data Events + +Events are sent to Fluvio as raw bytes. The event records are sent along to Fluvio as well. + diff --git a/content/connectors-old/inbound/mqtt.md b/content/connectors-old/inbound/mqtt.md new file mode 100644 index 00000000..57b3395e --- /dev/null +++ b/content/connectors-old/inbound/mqtt.md @@ -0,0 +1,74 @@ +--- +title: Inbound MQTT Connector +menu: MQTT +connector: + name: "infinyon/fluvio-connect-mqtt-source" + link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/mqtt" +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +MQTT is a publish/subscribe protocol that allows clients to listen to a stream +of events produced to a server. + +It is widely used in real-time and IoT applications +since it is lightweight and easy to use. + +## Common config values + +%copy% +```yaml +type: mqtt-source +``` + +%copy% +```yaml +version: 0.5.3 +``` + +## Parameters + +The inbound MQTT connector supports the following configuration options: + +### `mqtt-url` +*required* + +The hostname of the MQTT server to subscribe to + +This value can be provided as a parameter or as a secret + +### `mqtt-topic` +*required* + +The topic filter to use when receiving MQTT events + +#### Example connector config + +{{}} + +## Data Events + +There are two important pieces of information that we're interested in capturing from +every MQTT event. + +1. The topic the event was sent to +2. The body of the event + +In MQTT, topics are more of a label for particular message types. When we specify a +`mqtt-topic` to subscribe to, we are actually defining a "filter", or a pattern that +tells MQTT which events we are interested in receiving. + +This means that events we receive +may not all belong to the same topic - rather, they all match the filter we provided. + +The result of the event message body produced to Fluvio by the inbound MQTT connector is a JSON object containing: +* MQTT topic name +* MQTT message body + +Furthermore, since MQTT allows events to contain arbitrary +binary data, the payload is encoded as a byte buffer. + +Below is a sample of what an MQTT event captured and sent to Fluvio looks like: + +```json +{"mqtt_topic":"/hfp/v2/journey/ongoing/vp/bus/0022/01151/2200/1/Espoon keskus/23:02/1160105/4/60;24/29/00/00","payload":[123,34,86,80,34,58,123,34,100,101,115,105,34,58,34,50,48,48,34,44,34,100,105,114,34,58,34,49,34,44,34,111,112,101,114,34,58,50,50,44,34,118,101,104,34,58,49,49,53,49,44,34,116,115,116,34,58,34,50,48,50,49,45,49,49,45,49,56,84,50,49,58,49,51,58,51,50,46,56,56,52,90,34,44,34,116,115,105,34,58,49,54,51,55,50,55,48,48,49,50,44,34,115,112,100,34,58,49,50,46,54,55,44,34,104,100,103,34,58,51,51,55,44,34,108,97,116,34,58,54,48,46,50,48,48,49,53,55,44,34,108,111,110,103,34,58,50,52,46,57,48,48,50,49,53,44,34,97,99,99,34,58,45,48,46,49,49,44,34,100,108,34,58,45,55,53,44,34,111,100,111,34,58,110,117,108,108,44,34,100,114,115,116,34,58,110,117,108,108,44,34,111,100,97,121,34,58,34,50,48,50,49,45,49,49,45,49,56,34,44,34,106,114,110,34,58,49,53,48,44,34,108,105,110,101,34,58,49,48,52,52,44,34,115,116,97,114,116,34,58,34,50,51,58,48,50,34,44,34,108,111,99,34,58,34,71,80,83,34,44,34,115,116,111,112,34,58,110,117,108,108,44,34,114,111,117,116,101,34,58,34,50,50,48,48,34,44,34,111,99,99,117,34,58,48,125,125]} +``` diff --git a/content/connectors/inbound/postgres.md b/content/connectors-old/inbound/postgres.md similarity index 98% rename from content/connectors/inbound/postgres.md rename to content/connectors-old/inbound/postgres.md index bd89f16b..e86baa9e 100644 --- a/content/connectors/inbound/postgres.md +++ b/content/connectors-old/inbound/postgres.md @@ -93,7 +93,7 @@ This should contain #### Example connector config -{{}} +{{}} ## Data Events diff --git a/content/connectors/inbound/prometheus.md b/content/connectors-old/inbound/prometheus.md similarity index 100% rename from content/connectors/inbound/prometheus.md rename to content/connectors-old/inbound/prometheus.md diff --git a/content/connectors/inbound/template.md b/content/connectors-old/inbound/template.md similarity index 96% rename from content/connectors/inbound/template.md rename to content/connectors-old/inbound/template.md index 6c2fbc50..13923a85 100644 --- a/content/connectors/inbound/template.md +++ b/content/connectors-old/inbound/template.md @@ -5,6 +5,7 @@ section: Source toc: false hidden: true --- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} The TEMPLATE connector is a Source, which reads from an external data provider and produces data to Fluvio. diff --git a/content/connectors/local-connectors.md b/content/connectors-old/local-connectors.md similarity index 94% rename from content/connectors/local-connectors.md rename to content/connectors-old/local-connectors.md index 4305e19d..e2278ec7 100644 --- a/content/connectors/local-connectors.md +++ b/content/connectors-old/local-connectors.md @@ -2,6 +2,7 @@ title: Local Connectors weight: 20 --- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} Local connectors are provided as Docker containers. Using our `connector-run` utility supports starting connectors in Kubernetes or Docker using a connector config file. @@ -32,9 +33,9 @@ $ git clone https://github.com/infinyon/fluvio-connectors.git ``` ### Example HTTP connector -This is the config file for the [Inbound HTTP connector]({{}}) in this guide. +This is the config file for the [Inbound HTTP connector]({{}}) in this guide. -{{}} +{{}} In this config, we are creating a connector named `cat-facts`. It will request data from {{}} once every 30 seconds and receive json data. The connector will store the json into a topic called `cat-facts-data` diff --git a/content/connectors-old/outbound/_index.md b/content/connectors-old/outbound/_index.md new file mode 100644 index 00000000..38de8ec1 --- /dev/null +++ b/content/connectors-old/outbound/_index.md @@ -0,0 +1,6 @@ +--- +title: OUTBOUND +weight: 20 +_build: + render: never +--- diff --git a/content/connectors/outbound/dynamodb.md b/content/connectors-old/outbound/dynamodb.md similarity index 95% rename from content/connectors/outbound/dynamodb.md rename to content/connectors-old/outbound/dynamodb.md index 12f0be4f..527c04c1 100644 --- a/content/connectors/outbound/dynamodb.md +++ b/content/connectors-old/outbound/dynamodb.md @@ -67,7 +67,7 @@ The [AWS Secret Access Key] #### Example connector config -{{}} +{{}} [Dynamodb data types]: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBMapper.DataTypes.html [attribute types of the keys]: https://docs.aws.amazon.com/cli/latest/reference/dynamodb/create-table.html#options diff --git a/content/connectors-old/outbound/kafka.md b/content/connectors-old/outbound/kafka.md new file mode 100644 index 00000000..d8083e15 --- /dev/null +++ b/content/connectors-old/outbound/kafka.md @@ -0,0 +1,84 @@ +--- +title: Outbound Kafka Connector +menu: Kafka +connector: + name: "infinyon/fluvio-connect-kafka-sink" + link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sinks/kafka" +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +The Kafka Connector is quite simple. It will send every record of a fluvio topic to a kafka cluster for a given kafka topic. + +## Common config values + +%copy% +```yaml +type: kafka-sink +``` + +%copy% +```yaml +version: 0.3.3 +``` + +## Parameters + +### `kakfa-url` +*required* + +The url of the Kafka instance to connect to. + +### `kafka-topic` +Default: Same name as the configured `fluvio` topic + +The name of the Kafka topic name to connect to. + + +### `kafka-partition` +Default: `0` + +The Kafka partition to connect to. + +### `kafka-option` +*optional* + +option that is specified as a dictonary. + +See: The [Configuration +properties are from the rdkafka +options](https://github.com/edenhill/librdkafka/blob/b171d8f411a981c7604a79777ce10245f05280dd/CONFIGURATION.md). + +### `security` + +#### `security-protocol` +*optional* + +The kafka security protocol. + +Currently only supports `SSL`. + +## Secrets + +### `FLUVIO_KAFKA_CLIENT_KEY` +*optional* + +The SSL key pem text. + +### `FLUVIO_KAFKA_CLIENT_CERT` +*optional* + +The SSL cert pem text + +### `FLUVIO_KAFKA_CLIENT_CA` +*optional* + +The SSL CA pem text + +#### Example connector config + +{{}} + + +## Data Events + +Events are sent to Fluvio as raw bytes. The event records are sent along to Fluvio as well. diff --git a/content/connectors/outbound/postgres.md b/content/connectors-old/outbound/postgres.md similarity index 100% rename from content/connectors/outbound/postgres.md rename to content/connectors-old/outbound/postgres.md diff --git a/content/connectors/outbound/prometheus.md b/content/connectors-old/outbound/prometheus.md similarity index 100% rename from content/connectors/outbound/prometheus.md rename to content/connectors-old/outbound/prometheus.md diff --git a/content/connectors/outbound/slack.md b/content/connectors-old/outbound/slack.md similarity index 91% rename from content/connectors/outbound/slack.md rename to content/connectors-old/outbound/slack.md index 78feef13..4c3bc525 100644 --- a/content/connectors/outbound/slack.md +++ b/content/connectors-old/outbound/slack.md @@ -39,7 +39,7 @@ https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX #### Example connector config -{{}} +{{}} ## Data Events diff --git a/content/connectors-old/outbound/sql.md b/content/connectors-old/outbound/sql.md new file mode 100644 index 00000000..22c536f3 --- /dev/null +++ b/content/connectors-old/outbound/sql.md @@ -0,0 +1,110 @@ +--- +title: Outbound SQL Connector +menu: SQL +connector: + name: "infinyon/fluvio-connect-sql-sink" + link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sinks/sql" +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +-> SQL Connector is a preview with support for SmartModule chaining through `transforms`.
              See [HTTP-to-SQL]({{}}) and [MQTT-to-SQL]({{}}) for examples of this connector in action. + +The Outbound SQL connector supports PostgreSQL and SQLite databases. + +## Common config values + +%copy% +```yaml +type: sql-sink +``` + +%copy% +```yaml +version: 0.1.2 +``` + +## Parameters + +### `database_url` +*required* + +The connection string for a PostgreSQL or SQLite database + +### Data types + +This is the a table of what input types map to in PostgreSQL and SQLite + +| Model | PostgreSQL | SQLite | +|:----------------|:-----------------------------|:-------------| +| Bool | BOOL | BOOLEAN | +| Char | CHAR | INTEGER | +| SmallInt | SMALLINT, SMALLSERIAL, INT2 | INTEGER | +| Int | INT, SERIAL, INT4 | INTEGER | +| BigInt | BIGINT, BIGSERIAL, INT8 | BIGINT, INT8 | +| Float | REAL, FLOAT4 | REAL | +| DoublePrecision | DOUBLE PRECISION, FLOAT8 | REAL | +| Text | VARCHAR, CHAR(N), TEXT, NAME | TEXT | +| Bytes | BYTEA | BLOB | +| Numeric | NUMERIC | REAL | +| Timestamp | TIMESTAMP | DATETIME | +| Date | DATE | DATE | +| Time | TIME | TIME | +| Uuid | UUID | BLOB, TEXT | +| Json | JSON, JSONB | TEXT | + +## Transforms +*required* + +### Uses +This is the name of a SmartModules from the SmartModule Hub. + +`/@` + +### Invoke +This is the action + +Choices: +- `insert` +- `map` + +### With +These are parameters to the SmartModule + +## Example + +Transform with [Jolt SmartModule](https://github.com/infinyon/fluvio-connectors/blob/308ca0ec6e195210a86724ff8b0a32f6897c7b93/smartmodules/jolt/) +```yaml + - uses: infinyon/jolt@0.1.0 + invoke: insert + with: + spec: + - operation: shift + spec: + payload: + device: 'device' + - operation: default + spec: + device: + type: 'mobile' +``` + +Transform with [json-sql SmartModule](https://github.com/infinyon/fluvio-connectors/blob/308ca0ec6e195210a86724ff8b0a32f6897c7b93/smartmodules/json-sql) +```yaml + - uses: infinyon/json-sql@0.1.0 + invoke: insert + with: + mapping: + table: 'topic_message' + map-columns: + 'device_id': + json-key: 'device.device_id' + value: + type: 'int' + default: '0' + required: true + 'record': + json-key: '$' + value: + type: 'jsonb' + required: true +``` diff --git a/content/connectors/_index.md b/content/connectors/_index.md index c36df9ef..cc8baa80 100644 --- a/content/connectors/_index.md +++ b/content/connectors/_index.md @@ -10,40 +10,10 @@ You can import data with an `Inbound` connector and export data with an `Outboun Inbound and outbound connectors fundamentally work in the same way. The only difference is the direction your data is streaming with respect to a Fluvio topic. -There are 4 steps to the connector: - -Smart Connectors -- **Protocol**: Parses data according to the wire format of the connected data platform. -- **Extract**: Extracts raw data from the protocol format and packages it neatly into data structures - that may be used by subsequent stages or be produced directly to a topic. -- **Filter** (optional): A user-provided SmartModule that may determine whether a given record - should be discarded before sending it over the network to Fluvio, saving bandwidth. -- **Shape** (optional): A user-provided SmartModule that may take the extracted data structures and - transform them in to an application-specific format. - -The **Protocol** and **Extract** stages are built directly into the -connector. They offer basic access to your data through the various protocols your data sources use. - -In the **Extract** stage, your data is structured from whatever protocol it is sourced from. - -Additionally, You can apply custom pre-processing or post-processing to data, before it -arrives to or while it streams from a Fluvio topic. The **Filter** and **Shape** stages are provided through SmartModules. - -Powered by WebAssembly (also called wasm), SmartModules are pre-packaged or user-provided operations such as filters, maps, or aggregators that can be applied to records at various points in the streaming pipeline. -Supporting access to your data while it is in transit provides you the ability to clean, transform and enrich your data before it is stored in a topic, or it exits the Fluvio cluster. - -Use Connectors either as: -* a [Local Connector]({{}}) - * Run your connector on your machine as a docker container - -* a [Cloud Connector]({{}}), - * You can start a connector on [InfinyOn Cloud], and let us manage the infrastruture - -You can customize how your connectors run through a configuration file. - -For more info about connectors or configuration, check out our supported Inbound and Outbound connector docs. - -[InfinyOn Cloud]: https://infinyon.cloud +* Learn more about the connector's 4 steps of transformation in [core concepts]({{}}) +* To get started with using connectors, check out the [requirements]({{}}) first before browsing through one of our Inbound or Outbound connectors. +* For the development of your own connectors, continue on to the [Connector Development Kit docs]({{}}) diff --git a/content/connectors/cdk/_index.md b/content/connectors/cdk/_index.md new file mode 100644 index 00000000..ecef7c5b --- /dev/null +++ b/content/connectors/cdk/_index.md @@ -0,0 +1,6 @@ +--- +title: DEVELOPMENT +weight: 40 +_build: + render: never +--- \ No newline at end of file diff --git a/content/connectors/cdk/build-test.md b/content/connectors/cdk/build-test.md new file mode 100644 index 00000000..d17aa3e6 --- /dev/null +++ b/content/connectors/cdk/build-test.md @@ -0,0 +1,51 @@ +--- +title: Build & Test +weight: 40 +--- + +##### Prerequisites + +This section assumes that CDK is [installed]({{< ref "install" >}}) and `my-connector` project has been [generated]({{< ref "generate" >}}). + +### Build - Operation + +Navigate to `my-connector` project directory and run `build` + +%copy first-line% +```bash +$ cdk build +... +Compiling my-connector v0.1.0 (~/cdk/my-connector) + Finished release [optimized] target(s) in 1m 12s +``` + +The build process generated a binary for your custom connector. We are now ready to test it. + +## Test - Operation + +If the connector builds successfully, it’s time to start an instance with `cdk test`: + +The `--config ` argument is required. It is a path to the configuration file in YAML format. + +Generating the project with `cdk generate` should have created a sample config for you named `config-example.yaml` + +%copy first-line% +```bash +$ cdk test --config config-example.yaml + Finished release [optimized] target(s) in 0.16s +Connector runs with process id: 80380 +Starting my-connector source connector with CustomConfig { foo: "bar" } +``` + +SmartConnector output will be redirected to the current terminal output. + +To stop running SmartConnector in test mode, press Ctrl+C. + +### Steps + +1. [Install CDK]({{< ref "install" >}}) +2. [Generate a SmartConnector]({{< ref "generate" >}}) +3. **[Build and Test]({{< ref "build-test" >}})** +4. [Start and Shutdown]({{< ref "start-shutdown" >}}) +5. [List and Logs]({{< ref "list-log" >}}) +6. [Publish to SmartConnector Hub]({{< ref "publish" >}}) \ No newline at end of file diff --git a/content/connectors/cdk/generate.md b/content/connectors/cdk/generate.md new file mode 100644 index 00000000..2ab9e1f3 --- /dev/null +++ b/content/connectors/cdk/generate.md @@ -0,0 +1,118 @@ +--- +title: Generate +weight: 30 +--- + +CDK generate helps developers build a sample SmartConnector project by answering a few simple questions. +Prerequisites + +This section assumes that CDK is installed. + +Use `cdk generate` to create a new connector project: + +%copy first-line% +```bash +$ cdk generate +🤷 Project Name: my-connector +🔧 Destination: ~/my-connector ... +🔧 project-name: my-connector ... +🔧 Generating template ... +✔ 🤷 Will your Connector be public? · false +🤷 Please set a group name: aj +✔ 🤷 Which type of Connector would you like [source/sink]? · source +Ignoring: /var/folders/r8/4x6_d2rn283946frzd1gc1pr0000gn/T/.tmptToFV3/cargo-generate.toml +[1/6] Done: Cargo.toml +[2/6] Done: Connector.toml +[3/6] Done: config-example.yaml +[4/6] Done: src/config.rs +[5/6] Done: src/main.rs +[6/6] Done: src +🔧 Moving generated files into: `~/my-connector`... +💡 Initializing a fresh Git repository +✨ Done! New project created ~/my-connector +``` + +The generator created Rust project ready to compile: + +```bash +$ tree +. +├── Cargo.toml +├── Connector.toml +├── config-example.yaml +└── src + ├── config.rs + └── main.rs + +2 directories, 5 files +``` + +This is a simple SmartConnector `my-connector`: + +```rust +mod config; +use config::CustomConfig; + + +use fluvio::{RecordKey, TopicProducer}; +use fluvio_connector_common::{ + connector, + Result +}; + +#[connector(source)] +async fn start(config: CustomConfig, producer: TopicProducer) -> Result<()> { + println!("Starting my-connector source connector with {config:?}"); + for i in 1..1000 { + let value = format!("Hello, Fluvio - {i}"); + producer.send(RecordKey::NULL, value).await?; + producer.flush().await?; + std::thread::sleep(std::time::Duration::from_millis(1000)); + } + Ok(()) +} +``` + +**Note** the `Connector.toml` file. This file contains Connector parameters required to load the file in the Cluster and publish it to SmartConnector Hub. + +%copy first-line% +```bash +$ cat Connector.toml +[package] +name = "my-connector" +group = "aj" +version = "0.1.0" +apiVersion = "0.1.0" +fluvio = "0.10.0" +description = "" +license = "Apache-2.0" +visibility = "private" + +[direction] + source = true + +[deployment] +binary = "my-connector" + +[custom.properties.foo] +title = "Foo" +description = "Foo" +type = "string" +``` + +#### Sections +* `package` is used to build the SmartConnector FQDN `aj/my-connector@0.1.0`, and the description to publish to Hub. The `group` name is equivalent to the package owner in the Hub. +* `direction` is used to declare the direction data flows through the connector, with respect to the Fluvio cluster. An inbound connector uses `source = true`, and and outbound connectdor uses `sink = true` +* `deployment` +* `custom.properties.foo` defines a user configuration key `foo` that can be used in the logic of the connector. + +The project is ready to build and test. Checkout the next section for instructions. + +### Steps + +1. [Install CDK]({{< ref "install" >}}) +2. **[Generate a SmartConnector]({{< ref "generate" >}})** +3. [Build and Test]({{< ref "build-test" >}}) +4. [Start and Shutdown]({{< ref "start-shutdown" >}}) +5. [List and Logs]({{< ref "list-log" >}}) +6. [Publish to SmartConnector Hub]({{< ref "publish" >}}) \ No newline at end of file diff --git a/content/connectors/cdk/install.md b/content/connectors/cdk/install.md new file mode 100644 index 00000000..92421390 --- /dev/null +++ b/content/connectors/cdk/install.md @@ -0,0 +1,25 @@ +--- +title: Install +weight: 20 +--- + +### Install + +CDK is an independent executable downloadable via [`Fluvio CLI`](https://www.fluvio.io/cli/utilities/install/): + +```bash +$ fluvio install cdk +``` + +The executable is installed in `~/.fluvio/bin` directory, the same directory as `fluvio`. The command should be immediately accessible at the prompt: + +{{% inline-embed file="embeds/cli/help/cdk.md" %}} + +### Steps + +1. **[Install CDK]({{< ref "install" >}})** +2. [Generate a SmartConnector]({{< ref "generate" >}}) +3. [Build and Test]({{< ref "build-test" >}}) +4. [Start and Shutdown]({{< ref "start-shutdown" >}}) +5. [List and Logs]({{< ref "list-log" >}}) +6. [Publish to SmartConnector Hub]({{< ref "publish" >}}) \ No newline at end of file diff --git a/content/connectors/cdk/list-log.md b/content/connectors/cdk/list-log.md new file mode 100644 index 00000000..5549c5da --- /dev/null +++ b/content/connectors/cdk/list-log.md @@ -0,0 +1,33 @@ +--- +title: List & Logs +weight: 60 +--- +##### Prerequisites + +This section assumes that CDK is [installed]({{< ref "install" >}}) and `my-connector` project has been [generated]({{< ref "generate" >}}). + +After you [start your connectors]({{}}), you can review the list of all connectors, and view their logs with these commands + +### List + +%copy first-line% +```bash +$ cdk deploy list + NAME STATUS + my-my-connector-test-connector Running +``` + +### Log + +%copy first-line% +```bash +$ cdk deploy log my-my-connector-test-connector +Starting my-connector source connector with CustomConfig { foo: "bar" } +``` + +1. [Install CDK]({{< ref "install" >}}) +2. [Generate a SmartConnector]({{< ref "generate" >}}) +3. [Build and Test]({{< ref "build-test" >}}) +4. [Start and Shutdown]({{< ref "start-shutdown" >}}) +5. **[List and Logs]({{< ref "list-log" >}})** +6. [Publish to SmartConnector Hub]({{< ref "publish" >}}) \ No newline at end of file diff --git a/content/connectors/cdk/overview.md b/content/connectors/cdk/overview.md new file mode 100644 index 00000000..1aa6de54 --- /dev/null +++ b/content/connectors/cdk/overview.md @@ -0,0 +1,25 @@ +--- +title: Overview +weight: 10 +--- + +## Intro + +Fluvio Connector Development Kit (CDK) is a command-line tool whose primary goal is to help developers build, test, deploy SmartConnectors, and publish them to the Hub. + +[Source code](https://github.com/infinyon/fluvio/tree/master/crates/cdk) + +## Installation + +CDK only supports connectors developed in the Rust programming language. Other languages will be added in future releases. + +### Requirements + +Install Rust compiler and Cargo. See [https://www.rust-lang.org/tools/install](https://www.rust-lang.org/tools/install) for installation instructions. + +1. **[Install CDK]({{< ref "install" >}})** +2. [Generate a SmartConnector]({{< ref "generate" >}}) +3. [Build and Test]({{< ref "build-test" >}}) +4. [Start and Shutdown]({{< ref "start-shutdown" >}}) +5. [List and Logs]({{< ref "list-log" >}}) +6. [Publish to SmartConnector Hub]({{< ref "publish" >}}) \ No newline at end of file diff --git a/content/connectors/cdk/publish.md b/content/connectors/cdk/publish.md new file mode 100644 index 00000000..9d92ddd1 --- /dev/null +++ b/content/connectors/cdk/publish.md @@ -0,0 +1,41 @@ +--- +title: Publish +weight: 70 +--- +##### Prerequisites + +This section assumes that CDK is [installed]({{< ref "install" >}}) and `my-connector` project has been [generated]({{< ref "generate" >}}). + + +### Publish + +The final step in this scenario would be publishing the Connector to the Hub - where other users can download and integrate it into their data pipelines. + +If run without arguments, it will pack everything needed into a package and push the package to the Hub. (`cdk build`must be executed beforehand) + +If you need to inspect the package before the push: + +%copy first-line% +```bash +% cdk publish --pack +Using hub https://hub-dev.infinyon.cloud +Creating package aj/my-connector@0.1.0 +.. fill out info in hub/package-meta.yaml +Package hub/my-connector-0.1.0.ipkg created +``` + +Check the file and then push. The file is a Tar Gzip archive. + +%copy first-line% +```bash +$ cdk publish --push hub/my-connector-0.1.0.ipkg +``` + +`publish` command uses `hub/package-meta.yml` file with the metadata needed for the Hub. If it doesn’t exist, `cdk` creates it for you. Then, you can modify it, and the changes will be picked up on the subsequent command execution. + +1. [Install CDK]({{< ref "install" >}}) +2. [Generate a SmartConnector]({{< ref "generate" >}}) +3. [Build and Test]({{< ref "build-test" >}}) +4. [Start and Shutdown]({{< ref "start-shutdown" >}}) +5. [List and Logs]({{< ref "list-log" >}}) +6. **[Publish to SmartConnector Hub]({{< ref "publish" >}})** diff --git a/content/connectors/cdk/start-shutdown.md b/content/connectors/cdk/start-shutdown.md new file mode 100644 index 00000000..05779b3f --- /dev/null +++ b/content/connectors/cdk/start-shutdown.md @@ -0,0 +1,51 @@ +--- +title: Start & Shutdown +weight: 50 +--- + +##### Prerequisites + +This section assumes that CDK is [installed]({{< ref "install" >}}) and `my-connector` project has been [generated]({{< ref "generate" >}}). + + +### Connector start + +[Testing]({{}}) your connector runs the process in the foreground. + +When you are ready, you can run `cdk deploy start` to run your connector in the background. + +%copy first-line% +```bash +$ cdk deploy start --config config-example.yaml +Log file: /private/tmp/my-connector/my-connector.log +Connector runs with process id: 88589 +``` + +{{}} +To start an official connector from a local `.ipkg` connector package file, you can provide it with the `--ipkg` option + +Example: + +%copy first-line% +```bash +$ cdk deploy start --ipkg infinyon-http-source-0.1.1.ipkg --config config-example.yaml +``` +{{}} + + +### Connector shutdown + +Conversely, when you want to stop running your connector, you can run `cdk deploy shutdown ` to stop the running process of your connector in background. + +You can access the connector name from the [list of your running connectors]({{< ref "list-log" >}}), or you can find the connector `name` in your config file. + +{{}} +This command exits quietly +{{}} + +1. [Install CDK]({{< ref "install" >}}) +2. [Generate a SmartConnector]({{< ref "generate" >}}) +3. [Build and Test]({{< ref "build-test" >}}) +4. **[Start and Shutdown]({{< ref "start-shutdown" >}})** +5. [List and Logs]({{< ref "list-log" >}}) +6. [Publish to SmartConnector Hub]({{< ref "publish" >}}) \ No newline at end of file diff --git a/content/connectors/cloud-connectors.md b/content/connectors/cloud-connectors.md index e4c80dc5..af57eab5 100644 --- a/content/connectors/cloud-connectors.md +++ b/content/connectors/cloud-connectors.md @@ -25,7 +25,7 @@ Check out the [`fluvio cloud` CLI docs]({{}}) for {{}} ### Example HTTP connector -This is the config file for the [Inbound HTTP connector]({{}}) in this guide. +This is the config file for the [Inbound HTTP connector]({{}}) in this guide. {{}} @@ -48,8 +48,8 @@ After the connector is created, you can list the connectors you've created, and %copy first-line% ```bash $ fluvio cloud connector list - NAME TYPE VERSION STATUS - cat-facts http-source 0.3.0 Running + NAME TYPE VERSION CDK STATUS + cat-facts http-source 0.1.0 V3 Running ``` #### Look at connector logs @@ -59,16 +59,28 @@ If there is a need to debug the behavior of a connector, the logs are available %copy first-line% ```bash $ fluvio cloud connector logs cat-facts -2022-10-23T07:36:36.915928Z INFO http_source: Starting HTTP source connector connector_version="0.3.0" git_hash="10ee08a94b7be7d91a31a01104b7f6e86e54b7d9" -2022-10-23T07:36:36.915981Z INFO http_source: interval=30s method=GET topic=cat-facts output_parts=body output_type=text endpoint=https://catfact.ninja/fact -2022-10-23T07:36:36.916165Z INFO fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c" -2022-10-23T07:36:36.920362Z INFO fluvio::fluvio: Connecting to Fluvio cluster fluvio_crate_version="0.12.14" fluvio_git_hash="" -2022-10-23T07:36:36.979270Z INFO connect: fluvio::sockets: connect to socket add=fluvio-sc-public:9003 -2022-10-23T07:36:37.025300Z INFO connect:connect_with_config: fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c" -2022-10-23T07:36:37.088073Z INFO connect:connect_with_config:connect: fluvio::sockets: connect to socket add=fluvio-sc-public:9003 -2022-10-23T07:36:37.494092Z INFO dispatcher_loop{self=MultiplexDisp(13)}: fluvio_socket::multiplexing: multiplexer terminated -2022-10-23T07:36:37.544828Z INFO http_source: Connected to Fluvio -2022-10-23T07:36:38.060832Z INFO run:create_serial_socket_from_leader{leader_id=0}:connect_to_leader{leader=0}:connect: fluvio::sockets: connect to socket add=fluvio-spu-main-0.acct-.svc.cluster.local:9005 +connector-startup infinyon/http-source@0.1.0 +2023-03-25T03:41:29.570294Z INFO surf::middleware::logger::native: sending request +2023-03-25T03:41:29.702213Z INFO surf::middleware::logger::native: request completed +2023-03-25T03:41:29.702258Z INFO connector_startup::startup: downloading package url="https://hub.infinyon.cloud/hub/v0/connector/pkg/infinyon/http-source/0.1.0" +2023-03-25T03:41:29.702290Z INFO surf::middleware::logger::native: sending request +2023-03-25T03:41:29.993001Z INFO surf::middleware::logger::native: request completed +2023-03-25T03:41:30.108220Z INFO connector_startup::startup: writing file file="connector.ipkg" +... checking package +2023-03-25T03:41:30.301199Z INFO connector_startup::startup: connector binary from package path="./http-source" +2023-03-25T03:41:30.301224Z INFO connector_startup::startup: Starting deployment +Connector runs with process id: 15 +2023-03-25T03:41:30.303333Z INFO http_source: Reading config file from: /home/fluvio/config.yaml +2023-03-25T03:41:30.303526Z INFO http_source: starting processing +2023-03-25T03:41:30.304337Z INFO fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c.cloud-dev.fluvio.io" +2023-03-25T03:41:30.308822Z INFO fluvio::fluvio: Connecting to Fluvio cluster fluvio_crate_version="0.16.0" fluvio_git_hash="8d4023ee0dc7735aaa0c823dd2b235662112f090" +2023-03-25T03:41:30.369634Z INFO connect: fluvio_socket::versioned: connect to socket add=fluvio-sc-public:9003 +2023-03-25T03:41:30.412895Z INFO connect:connect_with_config: fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c.cloud-dev.fluvio.io" +2023-03-25T03:41:30.473242Z INFO connect:connect_with_config:connect: fluvio_socket::versioned: connect to socket add=fluvio-sc-public:9003 +2023-03-25T03:41:30.582726Z INFO dispatcher_loop{self=MultiplexDisp(12)}: fluvio_socket::multiplexing: multiplexer terminated +2023-03-25T03:41:30.632722Z INFO fluvio_connector_common::monitoring: using metric path: /fluvio_metrics/connector.sock +2023-03-25T03:41:30.632795Z INFO fluvio_connector_common::monitoring: monitoring started +2023-03-25T03:41:31.172075Z INFO run:create_serial_socket_from_leader{leader_id=0}:connect_to_leader{leader=0}:connect: fluvio_socket::versioned: connect to socket add=fluvio-spu-main-0.acct-ce0c1782-ca61-4c54-a08c-3ba985524553.svc.cluster.local:9005 ``` #### View data in topic @@ -108,12 +120,12 @@ $ fluvio cloud connector delete cat-facts connector "cat-facts" deleted ``` -Deleting your connector will not delete the topic used by the connector. If you want to delete the topic, you can run `fluvio topic delete cat-facts-data` +Deleting your connector will not delete the topic used by the connector. If you want to delete the topic, you can run `fluvio topic delete cat-facts` %copy first-line% ```shell -$ fluvio topic delete cat-facts-data -topic "cat-facts-data" deleted +$ fluvio topic delete cat-facts +topic "cat-facts" deleted ``` ### Conclusion diff --git a/content/connectors/core-concepts.md b/content/connectors/core-concepts.md new file mode 100644 index 00000000..84421fa6 --- /dev/null +++ b/content/connectors/core-concepts.md @@ -0,0 +1,32 @@ +--- +title: Smart Connectors Core Concepts +menu: Core Concepts +weight: 10 +--- + +There are 4 steps to the connector: + +Smart Connectors + +- **Protocol**: Parses data according to the wire format of the connected data platform. +- **Extract**: Extracts raw data from the protocol format and packages it neatly into data structures + that may be used by subsequent stages or be produced directly to a topic. +- **Filter** (optional): A user-provided SmartModule that may determine whether a given record + should be discarded before sending it over the network to Fluvio, saving bandwidth. +- **Shape** (optional): A user-provided SmartModule that may take the extracted data structures and + transform them in to an application-specific format. + +The **Protocol** and **Extract** stages are built directly into the +connector. They offer basic access to your data through the various protocols your data sources use. + +In the **Extract** stage, your data is structured from whatever protocol it is sourced from. + +Additionally, You can apply custom pre-processing or post-processing to data, before it +arrives to or while it streams from a Fluvio topic. The **Filter** and **Shape** stages are provided through SmartModules. + +Powered by WebAssembly (also called wasm), [SmartModules]({{}}) are pre-packaged or user-provided operations such as filters, maps, or aggregators that can be applied to records at various points in the streaming pipeline. +Supporting access to your data while it is in transit provides you the ability to clean, transform and enrich your data before it is stored in a topic, or it exits the Fluvio cluster. + +Use Connectors locally with [`cdk`]({{}}) or start a connector on [InfinyOn Cloud](https://infinyon.cloud), and let us manage the infrastruture diff --git a/content/connectors/inbound/http.md b/content/connectors/inbound/http.md index 8bbc3620..8c332fb4 100644 --- a/content/connectors/inbound/http.md +++ b/content/connectors/inbound/http.md @@ -1,220 +1,5 @@ --- -title: Inbound HTTP Connector -menu: HTTP -connector: - name: "infinyon/fluvio-connect-http-source" - link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/http" +menu: HTTP --- -Fluvio's `http` connector allows you to periodically fetch data from an HTTP endpoint -and feed the HTTP response into a Fluvio topic. - -This is useful for monitoring APIs -continuously and building streaming applications that react to new or updated info. - - -Note that this connector is _not_ intended for streaming HTTP endpoints, it instead -periodically sends HTTP requests and collects the response as an event. - -## Common config values - -%copy% -```yaml -type: http-source -``` - -%copy% -```yaml -version: 0.4.2 -``` - -## Parameters - -The inbound HTTP connector supports the following configuration options: - -### `endpoint` -*required* - -The HTTP endpoint to send requests to - -### `body` -*optional* - -The body to use in the HTTP request to the endpoint - -### `user_agent` -default: `fluvio/http-source X.Y.Z` - -The HTTP User-Agent request header used in the HTTP request. - -### `method` -default: `GET` - -Choices: -- `GET` -- `PUT` -- `POST` -- `DELETE` - -### `interval` -default: `10s` - -A time period between sending requests to the endpoint. - -The time is formatted as an integer and a suffix. Supported suffixes: - -- `msec`, `ms` -- milliseconds -- `seconds`, `second`, `sec`, `s` -- `minutes`, `minute`, `min`, `m` -- `hours`, `hour`, `hr`, `h` -- `days`, `day`, `d` - -### `headers` -*optional* - -A list additional headers to include in the HTTP request. In the form `Key=Value` - -Example - -```yaml -headers: - - Keep-Alive=timeout=5,max=1000 - - Cache-Control=no-cache -``` - -### `output_parts` -default: `body` - -HTTP Response output Record parts - -Choices: -* `body` -* `full` - -### `output_type` -default: `text` - -HTTP Response output Record type - -Choices: -* `text` -* `json` - -#### Example connector config - -{{}} - -This example config for inbound HTTP connector config reads from the {{}} website, and stores the output in the `cat-facts` topic every `300` seconds - -## Data Events - -The HTTP connector generates one data event per HTTP response. The data depends on the endpoint you are accessing and can be formatted in multiple ways, as described below. - -### HTTP - Body - -By default, the HTTP connector produces the `body` of the HTTP response in JSON format: - -Example output: `output_parts: body` - -%copy first-line% -```bash -$ fluvio consume cat-facts -T -d -{"fact":"A cat almost never meows at another cat, mostly just humans. Cats typically will spit, purr, and hiss at other cats.","length":116} -{"fact":"In one stride, a cheetah can cover 23 to 26 feet (7 to 8 meters).","length":65} -{"fact":"Phoenician cargo ships are thought to have brought the first domesticated cats to Europe in about 900 BC.","length":105} -``` - - -### HTTP - Header/Body - -Use the `output_parts` parameter to produce full HTTP responses: - -Example output: `output_parts: full` - -%copy first-line% -```bash -$ fluvio consume cat-facts -T=1 -d -Consuming records starting 1 from the end of topic 'cat-facts' -HTTP/1.1 200 OK -server: nginx -date: Thu, 28 Apr 2022 14:05:43 GMT -content-type: application/json -transfer-encoding: chunked -connection: keep-alive -vary: Accept-Encoding -cache-control: no-cache, private -x-ratelimit-limit: 100 -x-ratelimit-remaining: 98 -access-control-allow-origin: * -set-cookie: XSRF-TOKEN=eyJpdiI6IkgvdngxWVlsRUpiaXN2WFJnQTZyU1E9PSIsInZhbHVlIjoiMlVhQThFZmxCLzFnbVBrYWVubjBqS29TUENydkNibFp1bS9CajJtQkZvdk8vWHoxNFV2YmlZbVZIeU5JRDltWHU1ekRFamtrc2Mya0JGdGtuZlFkTFlHcmIwUTFCWURBOTlJZ1dHcld0VjlJcVp3cW1LZ2Z4RU8vTVRLS0FRVWMiLCJtYWMiOiI1ZTAwZTEwODkzMjRiNzFmNWU5YzJmZDVlNTgzZmI5Y2VkZDY4ZDMyOGU2MjNmZjEwZmJkZjMzZmI0NTI1YjgwIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 16:05:43 GMT; path=/; samesite=lax -set-cookie: cat_facts_session=eyJpdiI6ImF4c0p2aWM4TkJnUXdnK2psRUlyVWc9PSIsInZhbHVlIjoiR0Rkc3NjTU45cnpUc09pdHNEWk5kOW9BRFNTWVVUWDI2bE9yWWM2TDh2OW5LS0Uwbks2cXluR2R0QnEyYmMreWJYMlorb09WNjVEMkZSRGF5NGhYUDF6WjBac2lwYTJ3dVhlV3o1bXpsMGNoZ1hvcXo0YTNRczNqM1pheVR6ZHAiLCJtYWMiOiJiYTc1ZjA3NWEwY2IzYzkyZmIxNzdiOGY1YjQxZGNlMzU2ZjdiMjYwYzhkZjI0ZWJhNjkxZTY4NWExMDg3NzgxIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 16:05:43 GMT; path=/; httponly; samesite=lax -x-frame-options: SAMEORIGIN -x-xss-protection: 1; mode=block -x-content-type-options: nosniff - -{"fact":"There are more than 500 million domestic cats in the world, with approximately 40 recognized breeds.","length":100} -``` - -{{}} -The `header` is text and `body` is JSON. -{{}} - -### HTTP - JSON - -Set the `output_type` parameter to `json` if you want to convert the full HTTP response into JSON format. - - The following is the example output from {{}} with the following inbound HTTP connector config: -* `output_parts: full` -* `output_type: json` - -%copy first-line% -```bash -$ fluvio consume cat-facts -T=1 -d | jq -{ - "status": { - "version": "HTTP/1.1", - "code": 200, - "string": "OK" - }, - "header": { - "server": "nginx", - "date": "Thu, 28 Apr 2022 15:53:48 GMT", - "vary": "Accept-Encoding", - "cache-control": "no-cache, private", - "x-ratelimit-limit": "100", - "content-type": "application/json", - "x-ratelimit-remaining": "97", - "connection": "keep-alive", - "set-cookie": [ - "XSRF-TOKEN=eyJpdiI6IlFnMHN0UzFpMlBEaDhjdjBaRzNoN1E9PSIsInZhbHVlIjoibElGczZPaUFBdDc4eDhiT3N5bkJxZkRVOFpVTjdJaXI0VFFGS1dkU1ZodDV5SGVHOEF6bkhVWjFjTEduMWpuRU50WkV2WURML0JyWWdqWnBtWFpyUXJRU1BBU2Q1cTA1YmdMaWp4TGViRjJnVzNaMFFFdDBaVDhPOHVlRTQxTXUiLCJtYWMiOiJmMGJmY2Q1YmJmYjAwNTQ4NzM0ZmU0MzUxMGEzYzNiMDQ4NGIyOTE1ZjJiY2NkZjFlOWQzZGZmMDBiMWMwMmUwIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 17:53:48 GMT; path=/; samesite=lax", - "cat_facts_session=eyJpdiI6IklxdW1obmdqSFFSZ2ZRTmZGRnFNbXc9PSIsInZhbHVlIjoiSHJEQUJvZVVQbmJDNm9WaG5rbzNtYzBKZVN6Q0UwSGpXQTBRRHNxWGtVeVQ4bGxjU0lSZlhxa3g4SW0yS0k3bXpqNTIrK3R5K1Z3Vms5ck80dkdQWnlJWHhqWGpIY1R3Z25MWTBqTHAwZFlpQURGZUNFb2xRdmMwajVMaXV1TWkiLCJtYWMiOiJiN2Q3Yjk5MWE3Y2Q4ZWRkODk4MjM3YmM2NWJhZDI1MmY3ZmU0MDlkOWIyYzM0Zjg3YmE1NjdhNGMwZjliNjcyIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 17:53:48 GMT; path=/; httponly; samesite=lax" - ], - "x-xss-protection": "1; mode=block", - "x-content-type-options": "nosniff", - "transfer-encoding": "chunked", - "access-control-allow-origin": "*", - "x-frame-options": "SAMEORIGIN" - }, - "body": "{\"fact\":\"Most cats had short hair until about 100 years ago, when it became fashionable to own cats and experiment with breeding.\",\"length\":120}" -} -``` - --> **JSON Output** "body" is encoded in quotes (\") as JSON String within due to HTTP Response in this example containing application/json body itself. - -To convert only the body of the HTTP Response and ignore the header, set `output_parts` to `body` and the `output_type` keep as `json`: - -```json -{ - "body": "{\"fact\":\"Most cats had short hair until about 100 years ago, when it became fashionable to own cats and experiment with breeding.\",\"length\":120}" -} -``` - - -## Changelog - -| Version | Date | PR | Subject | -|:-------:|:----------:|:----------------------------------------------------------------:| ------------------------------------------------- | -| 0.3.0 | 2022-02-01 | [278](https://github.com/infinyon/fluvio-connectors/pull/278) |
              • Add interval times in human readable times
              • Support for producer and consumer top level config options
                  • | -| 0.2.0 | 2022-02-01 | [141](https://github.com/infinyon/fluvio-connectors/pull/141) |
                  • Feature json Response Type Record
                  • Deprecate metadata output_format in favor of `output_parts` [ body (default)
                  • Add Metadata `output_type` [ text (default) | json ] | full ]
                      • | -| 0.1.1 | 2022-01-31 | [127](https://github.com/infinyon/fluvio-connectors/pull/127) |
                      • Feature full Response Parts Record
                      • Add Metadata `output_format` [ body (default) | full ]
                          • | -| 0.1.0 | 2021-11-09 | - |
                          • Initial version with text/body Response (default) Record
                          | +{{% inline-embed file="embeds/connectors/inbound/http.md" %}} \ No newline at end of file diff --git a/content/connectors/inbound/kafka.md b/content/connectors/inbound/kafka.md index 9e03c1df..852055b1 100644 --- a/content/connectors/inbound/kafka.md +++ b/content/connectors/inbound/kafka.md @@ -1,58 +1,5 @@ --- -title: Inbound Kafka Connector -menu: Kafka -connector: - name: "infinyon/fluvio-connect-kafka-source" - link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/kafka" +menu: Kafka --- -The inbound Kafka Connector will send every record on a Kafka topic/partition to a Fluvio topic/partition. - -{{}} -The Inbound Kafka connector does not currently support SSL -{{}} - -## Common config values - -%copy% -```yaml -type: kafka-source -``` - -%copy% -```yaml -version: 0.3.1 -``` - -## Parameters - -The inbound Kafka connector supports the following configuration options: - -### `kakfa-url` -*required* - -The url of the Kafka instance to connect to. - -### `kafka-topic` -Default: Same name as the configured `fluvio` topic - -The name of the Kafka topic name to connect to. - -### `kafka-partition` -Default: `0` - -The Kafka partition to connect to. - -### `kafka-group` -Default: `fluvio-kafka-source` - -The Kafka group - -#### Example connector config - -{{}} - -## Data Events - -Events are sent to Fluvio as raw bytes. The event records are sent along to Fluvio as well. - +{{% inline-embed file="embeds/connectors/inbound/kafka.md" %}} \ No newline at end of file diff --git a/content/connectors/inbound/mqtt.md b/content/connectors/inbound/mqtt.md index af207eca..57a92a52 100644 --- a/content/connectors/inbound/mqtt.md +++ b/content/connectors/inbound/mqtt.md @@ -1,73 +1,5 @@ --- -title: Inbound MQTT Connector -menu: MQTT -connector: - name: "infinyon/fluvio-connect-mqtt-source" - link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/mqtt" +menu: MQTT --- -MQTT is a publish/subscribe protocol that allows clients to listen to a stream -of events produced to a server. - -It is widely used in real-time and IoT applications -since it is lightweight and easy to use. - -## Common config values - -%copy% -```yaml -type: mqtt-source -``` - -%copy% -```yaml -version: 0.5.2 -``` - -## Parameters - -The inbound MQTT connector supports the following configuration options: - -### `mqtt-url` -*required* - -The hostname of the MQTT server to subscribe to - -This value can be provided as a parameter or as a secret - -### `mqtt-topic` -*required* - -The topic filter to use when receiving MQTT events - -#### Example connector config - -{{}} - -## Data Events - -There are two important pieces of information that we're interested in capturing from -every MQTT event. - -1. The topic the event was sent to -2. The body of the event - -In MQTT, topics are more of a label for particular message types. When we specify a -`mqtt-topic` to subscribe to, we are actually defining a "filter", or a pattern that -tells MQTT which events we are interested in receiving. - -This means that events we receive -may not all belong to the same topic - rather, they all match the filter we provided. - -The result of the event message body produced to Fluvio by the inbound MQTT connector is a JSON object containing: -* MQTT topic name -* MQTT message body - -Furthermore, since MQTT allows events to contain arbitrary -binary data, the payload is encoded as a byte buffer. - -Below is a sample of what an MQTT event captured and sent to Fluvio looks like: - -```json -{"mqtt_topic":"/hfp/v2/journey/ongoing/vp/bus/0022/01151/2200/1/Espoon keskus/23:02/1160105/4/60;24/29/00/00","payload":[123,34,86,80,34,58,123,34,100,101,115,105,34,58,34,50,48,48,34,44,34,100,105,114,34,58,34,49,34,44,34,111,112,101,114,34,58,50,50,44,34,118,101,104,34,58,49,49,53,49,44,34,116,115,116,34,58,34,50,48,50,49,45,49,49,45,49,56,84,50,49,58,49,51,58,51,50,46,56,56,52,90,34,44,34,116,115,105,34,58,49,54,51,55,50,55,48,48,49,50,44,34,115,112,100,34,58,49,50,46,54,55,44,34,104,100,103,34,58,51,51,55,44,34,108,97,116,34,58,54,48,46,50,48,48,49,53,55,44,34,108,111,110,103,34,58,50,52,46,57,48,48,50,49,53,44,34,97,99,99,34,58,45,48,46,49,49,44,34,100,108,34,58,45,55,53,44,34,111,100,111,34,58,110,117,108,108,44,34,100,114,115,116,34,58,110,117,108,108,44,34,111,100,97,121,34,58,34,50,48,50,49,45,49,49,45,49,56,34,44,34,106,114,110,34,58,49,53,48,44,34,108,105,110,101,34,58,49,48,52,52,44,34,115,116,97,114,116,34,58,34,50,51,58,48,50,34,44,34,108,111,99,34,58,34,71,80,83,34,44,34,115,116,111,112,34,58,110,117,108,108,44,34,114,111,117,116,101,34,58,34,50,50,48,48,34,44,34,111,99,99,117,34,58,48,125,125]} -``` +{{% inline-embed file="embeds/connectors/inbound/mqtt.md" %}} \ No newline at end of file diff --git a/content/connectors/outbound/_index.md b/content/connectors/outbound/_index.md index 38de8ec1..59c7c738 100644 --- a/content/connectors/outbound/_index.md +++ b/content/connectors/outbound/_index.md @@ -1,6 +1,6 @@ --- title: OUTBOUND -weight: 20 +weight: 10 _build: render: never --- diff --git a/content/connectors/outbound/kafka.md b/content/connectors/outbound/kafka.md index 260a7df7..a3e2adfb 100644 --- a/content/connectors/outbound/kafka.md +++ b/content/connectors/outbound/kafka.md @@ -1,83 +1,5 @@ --- -title: Outbound Kafka Connector -menu: Kafka -connector: - name: "infinyon/fluvio-connect-kafka-sink" - link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sinks/kafka" +menu: Kafka --- -The Kafka Connector is quite simple. It will send every record of a fluvio topic to a kafka cluster for a given kafka topic. - -## Common config values - -%copy% -```yaml -type: kafka-sink -``` - -%copy% -```yaml -version: 0.3.2 -``` - -## Parameters - -### `kakfa-url` -*required* - -The url of the Kafka instance to connect to. - -### `kafka-topic` -Default: Same name as the configured `fluvio` topic - -The name of the Kafka topic name to connect to. - - -### `kafka-partition` -Default: `0` - -The Kafka partition to connect to. - -### `kafka-option` -*optional* - -option that is specified as a dictonary. - -See: The [Configuration -properties are from the rdkafka -options](https://github.com/edenhill/librdkafka/blob/b171d8f411a981c7604a79777ce10245f05280dd/CONFIGURATION.md). - -### `security` - -#### `security-protocol` -*optional* - -The kafka security protocol. - -Currently only supports `SSL`. - -## Secrets - -### `FLUVIO_KAFKA_CLIENT_KEY` -*optional* - -The SSL key pem text. - -### `FLUVIO_KAFKA_CLIENT_CERT` -*optional* - -The SSL cert pem text - -### `FLUVIO_KAFKA_CLIENT_CA` -*optional* - -The SSL CA pem text - -#### Example connector config - -{{}} - - -## Data Events - -Events are sent to Fluvio as raw bytes. The event records are sent along to Fluvio as well. +{{% inline-embed file="embeds/connectors/outbound/kafka.md" %}} \ No newline at end of file diff --git a/content/connectors/outbound/sql.md b/content/connectors/outbound/sql.md index e7cb951b..91c36471 100644 --- a/content/connectors/outbound/sql.md +++ b/content/connectors/outbound/sql.md @@ -1,109 +1,5 @@ --- -title: Outbound SQL Connector menu: SQL -connector: - name: "infinyon/fluvio-connect-sql-sink" - link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sinks/sql" --- --> SQL Connector is a preview with support for SmartModule chaining through `transforms`.
                          See [HTTP-to-SQL]({{}}) and [MQTT-to-SQL]({{}}) for examples of this connector in action. - -The Outbound SQL connector supports PostgreSQL and SQLite databases. - -## Common config values - -%copy% -```yaml -type: sql-sink -``` - -%copy% -```yaml -version: 0.1.1 -``` - -## Parameters - -### `database_url` -*required* - -The connection string for a PostgreSQL or SQLite database - -### Data types - -This is the a table of what input types map to in PostgreSQL and SQLite - -| Model | PostgreSQL | SQLite | -|:----------------|:-----------------------------|:-------------| -| Bool | BOOL | BOOLEAN | -| Char | CHAR | INTEGER | -| SmallInt | SMALLINT, SMALLSERIAL, INT2 | INTEGER | -| Int | INT, SERIAL, INT4 | INTEGER | -| BigInt | BIGINT, BIGSERIAL, INT8 | BIGINT, INT8 | -| Float | REAL, FLOAT4 | REAL | -| DoublePrecision | DOUBLE PRECISION, FLOAT8 | REAL | -| Text | VARCHAR, CHAR(N), TEXT, NAME | TEXT | -| Bytes | BYTEA | BLOB | -| Numeric | NUMERIC | REAL | -| Timestamp | TIMESTAMP | DATETIME | -| Date | DATE | DATE | -| Time | TIME | TIME | -| Uuid | UUID | BLOB, TEXT | -| Json | JSON, JSONB | TEXT | - -## Transforms -*required* - -### Uses -This is the name of a SmartModules from the SmartModule Hub. - -`/@` - -### Invoke -This is the action - -Choices: -- `insert` -- `map` - -### With -These are parameters to the SmartModule - -## Example - -Transform with [Jolt SmartModule](https://github.com/infinyon/fluvio-connectors/blob/308ca0ec6e195210a86724ff8b0a32f6897c7b93/smartmodules/jolt/) -```yaml - - uses: infinyon/jolt@0.1.0 - invoke: insert - with: - spec: - - operation: shift - spec: - payload: - device: 'device' - - operation: default - spec: - device: - type: 'mobile' -``` - -Transform with [json-sql SmartModule](https://github.com/infinyon/fluvio-connectors/blob/308ca0ec6e195210a86724ff8b0a32f6897c7b93/smartmodules/json-sql) -```yaml - - uses: infinyon/json-sql@0.1.0 - invoke: insert - with: - mapping: - table: 'topic_message' - map-columns: - 'device_id': - json-key: 'device.device_id' - value: - type: 'int' - default: '0' - required: true - 'record': - json-key: '$' - value: - type: 'jsonb' - required: true -``` +{{% inline-embed file="embeds/connectors/outbound/sql.md" %}} \ No newline at end of file diff --git a/content/connectors/requirements.md b/content/connectors/requirements.md new file mode 100644 index 00000000..4e13f7f0 --- /dev/null +++ b/content/connectors/requirements.md @@ -0,0 +1,21 @@ +--- +title: System requirements +weight: 20 +--- + +CDK currently supports Rust programming language. Other languages will be added in future release. + +### Requirements + +Install Rust compiler and Cargo. + +See [https://www.rust-lang.org/tools/install](https://www.rust-lang.org/tools/install) for installation instructions. + +### Install + +CDK is an independent executable downloadable via [Fluvio CLI]({{}}): + +{{% inline-embed file="embeds/cli/install/cdk.md" %}} + +The executable is installed in `~/.fluvio/bin` directory, the same directory as `fluvio`. The command should be immediately accessible at the prompt: + diff --git a/content/docs/tutorials/data-pipeline.md b/content/docs/tutorials/data-pipeline.md index 0cdc9501..adcf0684 100644 --- a/content/docs/tutorials/data-pipeline.md +++ b/content/docs/tutorials/data-pipeline.md @@ -23,7 +23,7 @@ There are two main steps for this tutorial: We will be looking at the [Inbound HTTP Connector]({{}}) setup, and connecting to the catfact.ninja database to ingest and store JSON data into a topic. -The Outbound connector will be using a PostgreSQL database. It will listen to the topic for new records and insert them into a table. +The Outbound connector will be using a [PostgreSQL](https://www.postgresql.org/) database. It will listen to the topic for new records and insert them into a table. You can use your own PostgreSQL instance, if it can be reached over the internet. But you can still follow along by creating a PostgreSQL database at a hosting service, such as [ElephantSQL](https://www.elephantsql.com/). @@ -38,7 +38,7 @@ Fluvio can poll a multitude of input types. This is the template YAML connector file. To make it useful, it needs to be populated – which we will do in the next step. See -[the documentation]({{}}) for the parameters available for use. +[the documentation]({{}}) for the parameters available for use. {{}} @@ -53,8 +53,8 @@ polling, and the interval at which it polls. {{}} This creates a connector named `cat-facts`, that reads from the website -`https://catfact.ninja/fact` every 30 seconds, and produces to the topic -`cat-facts-data`. +`https://catfact.ninja/fact` every 10 seconds, and produces to the topic +`cat-facts`. #### Testing the Inbound Connector @@ -70,8 +70,8 @@ You can use `fluvio cloud connector list` to view the status of the connector. %copy first-line% ```shell $ fluvio cloud connector list - NAME TYPE VERSION STATUS - cat-facts http-source 0.4.2 Running + NAME TYPE VERSION CDK STATUS + cat-facts http-source 0.1.0 V3 Running ``` And `fluvio consume` to view the incoming data in the topic. @@ -155,7 +155,7 @@ $ fluvio hub download infinyon/jolt@0.1.0 For more info about the SmartModule Hub, check out the [Hub Overview page]({{}}) #### Outbound SQL with basic SQL inserts -In this connector, we will listen in on the `cat-facts-data` topic. Whenever a new fact is produced to the topic, the Outbound SQL connector will insert the record into a table named `animalfacts`. The length in one column called `length` and the entire JSON in another column `raw_fact_json`. +In this connector, we will listen in on the `cat-facts` topic. Whenever a new fact is produced to the topic, the Outbound SQL connector will insert the record into a table named `animalfacts`. The length in one column called `length` and the entire JSON in another column `raw_fact_json`. {{}} @@ -187,7 +187,7 @@ After a few seconds, we can see data in the PostgreSQL table, #### Outbound SQL with JSON to JSON transformation before insert -In this connector, we will listen in on the `cat-facts-data` topic. +In this connector, we will listen in on the `cat-facts` topic. But before we insert into the database, we specify a transformation. The resulting JSON we see inserted in the table has the `length` removed, and adds `type: cat` to every JSON. @@ -261,4 +261,4 @@ With the Outbound SQL Connector, we utilized SmartModules in two different ways. * [Connectors]({{}}) * [Smart Modules]({{}}) * [SmartModule Rust API](https://docs.rs/fluvio-smartmodule/latest/fluvio_smartmodule/) -* [Transformations]({{}}) +* [Transformations]({{}}): diff --git a/content/docs/tutorials/mqtt-to-sql.md b/content/docs/tutorials/mqtt-to-sql.md index f613ea52..e17b7089 100644 --- a/content/docs/tutorials/mqtt-to-sql.md +++ b/content/docs/tutorials/mqtt-to-sql.md @@ -10,7 +10,7 @@ We'll use 2 connectors: * [Outbound SQL connector]({{}}) * There will be an example of combining multiple SmartModules, known as **SmartModule chaining** -The Outbound connector will be using a PostgreSQL database. It will listen to the topic for new records and insert them into a table. +The Outbound connector will be using a [PostgreSQL](https://www.postgresql.org/) database. It will listen to the topic for new records and insert them into a table. You can use your own PostgreSQL instance, if it can be reached over the internet. But you can still follow along by creating a PostgreSQL database at a hosting service, such as [ElephantSQL](https://www.elephantsql.com/). diff --git a/content/news/this-week-in-fluvio-0009.md b/content/news/this-week-in-fluvio-0009.md index dbaf20fb..26986fc7 100644 --- a/content/news/this-week-in-fluvio-0009.md +++ b/content/news/this-week-in-fluvio-0009.md @@ -154,4 +154,4 @@ Until next week! [our CHANGELOG]: https://github.com/infinyon/fluvio/blob/master/CHANGELOG.md [our Discord channel]: https://discordapp.com/invite/bBG2dTz [Github Discussions]: https://github.com/infinyon/fluvio/discussions -[connectors]: {{}} \ No newline at end of file +[connectors]: {{}} \ No newline at end of file diff --git a/content/news/this-week-in-fluvio-0015.md b/content/news/this-week-in-fluvio-0015.md index e44cf084..d3b92fca 100644 --- a/content/news/this-week-in-fluvio-0015.md +++ b/content/news/this-week-in-fluvio-0015.md @@ -52,7 +52,7 @@ This feature was teased [last week], but now it is ready to be tried out. Check out the new [Connector Developer guide] for more information about how to create your own connectors. -[Connector Developer guide]: {{< ref "/connectors/how-to/overview.md">}} +[Connector Developer guide]: {{< ref "/connectors-old/how-to/overview.md">}} ### FilterMap SmartModule diff --git a/content/news/this-week-in-fluvio-0016.md b/content/news/this-week-in-fluvio-0016.md index a6c3cc6b..1a6cef0f 100644 --- a/content/news/this-week-in-fluvio-0016.md +++ b/content/news/this-week-in-fluvio-0016.md @@ -12,7 +12,7 @@ programmable streaming platform written in Rust. ## New Release - Fluvio v0.9.14 ### Connector logs -Logs from [connectors]({{}}) are now accessible from the CLI +Logs from [connectors]({{}}) are now accessible from the CLI %copy first-line% ```shell diff --git a/content/news/this-week-in-fluvio-0021.md b/content/news/this-week-in-fluvio-0021.md index de43a02e..72859300 100644 --- a/content/news/this-week-in-fluvio-0021.md +++ b/content/news/this-week-in-fluvio-0021.md @@ -15,7 +15,7 @@ This release was heavily focused on stability improvements ### Local Connector development fix -Previously, if you were [developing your own connector]({{}}), creating connectors would fail because during the creation of the connector pod, Kubernetes would always try to pull from Docker Hub. +Previously, if you were [developing your own connector]({{}}), creating connectors would fail because during the creation of the connector pod, Kubernetes would always try to pull from Docker Hub. You can control this behavior in your connector config through the `version` key. diff --git a/content/news/this-week-in-fluvio-0022.md b/content/news/this-week-in-fluvio-0022.md index 4b118340..962632cc 100644 --- a/content/news/this-week-in-fluvio-0022.md +++ b/content/news/this-week-in-fluvio-0022.md @@ -301,7 +301,7 @@ Consuming records from the end of topic 'cat-facts'. This will wait for new reco {"status":{"version":"HTTP/1.1","code":200,"string":"OK"},"header":{"set-cookie":["XSRF-TOKEN=REDACTED expires=Wed, 16-Feb-2022 02:56:22 GMT; path=/; samesite=lax","cat_facts_session=REDACTED expires=Wed, 16-Feb-2022 02:56:22 GMT; path=/; httponly; samesite=lax"],"content-type":"application/json","x-frame-options":"SAMEORIGIN","x-content-type-options":"nosniff","x-xss-protection":"1; mode=block","vary":"Accept-Encoding","server":"nginx","x-ratelimit-remaining":"94","date":"Wed, 16 Feb 2022 00:56:22 GMT","transfer-encoding":"chunked","cache-control":"no-cache, private","x-ratelimit-limit":"100","access-control-allow-origin":"*","connection":"keep-alive"},"body":"{\"fact\":\"There are more than 500 million domestic cats in the world, with approximately 40 recognized breeds.\",\"length\":100}"} ``` -[Updated docs for the HTTP Connector are available]({{}}) +[Updated docs for the HTTP Connector are available]({{}}) --- diff --git a/content/news/this-week-in-fluvio-0036.md b/content/news/this-week-in-fluvio-0036.md index c72ec754..1c6eaa6c 100644 --- a/content/news/this-week-in-fluvio-0036.md +++ b/content/news/this-week-in-fluvio-0036.md @@ -15,8 +15,8 @@ Please subscribe to This Week in Fluvio to receive new posts in your inbox ## New release * New Kafka Source and Sink connectors - * [Kafka Source docs]({{}}) - * [Kafka Sink docs]({{}}) + * [Kafka Source docs]({{}}) + * [Kafka Sink docs]({{}}) ## Developer experience improvements The Fluvio Java client is now [hosted in Maven Central](https://search.maven.org/artifact/com.infinyon/fluvio), which should reduce the friction for Java developers to install. @@ -32,7 +32,7 @@ We've added the capability to query your account's [CPU and memory usage via Clo ## Open positions * [Sr Rust Engineer (Frontend)](https://www.infinyon.com/careers/cloud-ui-engineer-senior-level) [Remote - Global] * [SW Engineer (Cloud services)](https://www.infinyon.com/careers/cloud-engineer-mid-level) [Remote - Global] -* [Sr SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-engineer-senior-level) [Remote - Global] +* [Sr SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-old-engineer-senior-level) [Remote - Global] * [Sr Rust engineer (Infrastructure)](https://www.infinyon.com/careers/infrastructure-engineer-senior-level) [Remote - US] * [Developer Advocate](https://www.infinyon.com/careers/developer-advocate-mid-senior-level) [Remote - US timezones +/- 5hr] * [Solutions Architect](https://www.infinyon.com/careers/solutions-architect) [Remote - US timezones +/- 5hr] diff --git a/content/news/this-week-in-fluvio-0048.md b/content/news/this-week-in-fluvio-0048.md index 4eb6d272..977ef311 100644 --- a/content/news/this-week-in-fluvio-0048.md +++ b/content/news/this-week-in-fluvio-0048.md @@ -19,12 +19,12 @@ Please subscribe to This Week in Fluvio to receive new posts in your inbox * [Fluvio v0.10.0](https://github.com/infinyon/fluvio/releases/tag/v0.10.0) * [`smdk` - SmartModule Development Kit CLI](https://www.fluvio.io/smartmodules/smdk/overview/) -* [SQL outbound connector](https://www.fluvio.io/connectors/outbound/sql/) +* [SQL outbound connector]({{}}) ### Deprecations The `fluvio connector` CLI and Fluvio's management of connectors has been removed in this release. -You can still use local connectors with your local Fluvio cluster. For more about local connectors see the [local connectors docs]({{< ref "/connectors/local-connectors" >}}) +You can still use local connectors with your local Fluvio cluster. For more about local connectors see the [local connectors docs]({{< ref "/connectors-old/local-connectors" >}}) ## New features @@ -43,7 +43,7 @@ The SmartModule Development kit reduces the number of steps required to get star [SmartModule Development Kit docs]({{< ref "/smartmodules/smdk/overview" >}}) ### SmartModule chaining preview -This release has a preview for SmartModule chaining. This functionality is offered with our [SQL outbound Cloud connector]({{< ref "/connectors/outbound/sql" >}}). +This release has a preview for SmartModule chaining. This functionality is offered with our [SQL outbound Cloud connector]({{< ref "/connectors-old/outbound/sql" >}}). To see it in action, you can follow the following tutorials: @@ -65,7 +65,7 @@ Check out the [New UI tutorial]({{< ref "/docs/tutorials/cloud-setup" >}}) for m ### Cloud connectors Management of connectors is now exclusive to InfinyOn Cloud. You can create connectors in InfinyOn with the `fluvio cloud connector` CLI. -Check out the [Cloud connectors docs]({{< ref "/connectors/cloud-connectors" >}}) for more info +Check out the [Cloud connectors docs]({{< ref "/connectors-old/cloud-connectors" >}}) for more info ### SmartModule Hub SmartModule Hub is a new service for offering public SmartModules. This removes the requirement of installing a SmartModule development environment in order to take advantage of SmartModules. You can download SmartModules from the Hub directly to your cluster to use. @@ -89,7 +89,7 @@ Thanks to those who were in attendance at KubeCon and stopped and said hi to us * [Head of Product Management](https://www.infinyon.com/careers/head-of-product-management) [Remote - Global] * [SW Engineer (Cloud services)](https://www.infinyon.com/careers/cloud-engineer-mid-level) [Remote - Global] * [Sr. Rust engineer (Infrastructure)](https://www.infinyon.com/careers/infrastructure-engineer-senior-level) [Remote - US] -* [Sr. SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-engineer-senior-level) [Remote - Global] +* [Sr. SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-old-engineer-senior-level) [Remote - Global] * [Developer Advocate](https://www.infinyon.com/careers/developer-advocate-mid-senior-level) [Remote - US timezones +/- 5hr] * [Solutions Architect](https://www.infinyon.com/careers/solutions-architect) [Remote - US timezones +/- 5hr] diff --git a/content/news/this-week-in-fluvio-0049.md b/content/news/this-week-in-fluvio-0049.md index 51eedc95..47bb4cae 100644 --- a/content/news/this-week-in-fluvio-0049.md +++ b/content/news/this-week-in-fluvio-0049.md @@ -18,7 +18,7 @@ Please subscribe to This Week in Fluvio to receive new posts in your inbox ## New release * [Fluvio v0.10.1](https://github.com/infinyon/fluvio/releases/tag/v0.10.1) -* [Kafka outbound connector v0.3.1]({{< ref "/connectors/outbound/kafka" >}}) +* [Kafka outbound connector v0.3.1]({{< ref "/connectors-old/outbound/kafka" >}}) ## New features @@ -28,7 +28,7 @@ Please subscribe to This Week in Fluvio to receive new posts in your inbox * SMDK: ([#2784](https://github.com/infinyon/fluvio/pull/2784)) * Fluvio CLI: ([#2812](https://github.com/infinyon/fluvio/pull/2812)) -SmartModule transformation chaining was [introduced in the last release]({{}}) as a preview with our [SQL outbound connector]({{}}). +SmartModule transformation chaining was [introduced in the last release]({{}}) as a preview with our [SQL outbound connector]({{}}). In this release, support is now available to the Rust client, `fluvio` and `smdk` CLI, and connectors wit the keyword `transforms`. @@ -38,7 +38,7 @@ To get familiar, check out the example configs from our tutorials. * [MQTT to SQL tutorial]({{< ref "/docs/tutorials/mqtt-to-sql" >}}) -* Kafka outbound connector now supports SSL - For more info check out the [Kafka outbound connector docs]({{< ref "/connectors/outbound/kafka" >}}) +* Kafka outbound connector now supports SSL - For more info check out the [Kafka outbound connector docs]({{< ref "/connectors-old/outbound/kafka" >}}) ## Bug fixes @@ -52,7 +52,7 @@ To get familiar, check out the example configs from our tutorials. * [Head of Product Management](https://www.infinyon.com/careers/head-of-product-management) [Remote - Global] * [SW Engineer (Cloud services)](https://www.infinyon.com/careers/cloud-engineer-mid-level) [Remote - Global] * [Sr. Rust engineer (Infrastructure)](https://www.infinyon.com/careers/infrastructure-engineer-senior-level) [Remote - US] -* [Sr. SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-engineer-senior-level) [Remote - Global] +* [Sr. SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-old-engineer-senior-level) [Remote - Global] * [Developer Advocate](https://www.infinyon.com/careers/developer-advocate-mid-senior-level) [Remote - US timezones +/- 5hr] * [Solutions Architect](https://www.infinyon.com/careers/solutions-architect) [Remote - US timezones +/- 5hr] diff --git a/content/smartmodules/certified/json-sql.md b/content/smartmodules/certified/json-sql.md index 2d1fe1fb..7d606557 100644 --- a/content/smartmodules/certified/json-sql.md +++ b/content/smartmodules/certified/json-sql.md @@ -158,4 +158,4 @@ For additional examples checkout the tutorials: * [Build HTTP to SQL Pipeline]({{}}) * [Build MQTT to SQL Pipeline]({{}}) -[sql-sink-connector]: {{}} +[sql-sink-connector]: {{}} diff --git a/content/smartmodules/smdk/build-test.md b/content/smartmodules/smdk/build-test.md index 4c12ad58..e407cd52 100644 --- a/content/smartmodules/smdk/build-test.md +++ b/content/smartmodules/smdk/build-test.md @@ -5,7 +5,7 @@ weight: 40 toc: false --- -SmartModules are binaries developed for real-time data stream processing, requiring in-line and live testing facilities. This section covers in-line testing, and the [next section] describes how to load SmartMoudules on the Cluster for live traffic testing. +SmartModules are binaries developed for real-time data stream processing, requiring in-line and live testing facilities. This section covers in-line testing, and the [next section] describes how to load SmartModules on the Cluster for live traffic testing. ##### Prerequisites @@ -27,7 +27,7 @@ The build process generated an binary optimized for WebAssembly. We are now read ### Test - Operation -Testing a SmartModule takes `text` of `files` from the commnad line. +Testing a SmartModule takes `text` of `files` from the command line. Let's perform the matching test for our `my-filter` that looks for letter `a`: diff --git a/content/smartmodules/smdk/generate.md b/content/smartmodules/smdk/generate.md index bc6a0bd7..82c69f74 100644 --- a/content/smartmodules/smdk/generate.md +++ b/content/smartmodules/smdk/generate.md @@ -5,15 +5,15 @@ weight: 30 toc: false --- -SMDK generate helps develpoers build a sample SmartModule project by answering a few simple quuestions. +SMDK generate helps developers build a sample SmartModule project by answering a few simple questions ##### Prerequisites -This section assumes that SMDK is [installed]. +This section assumes that SMDK is [installed] ### Generate - Operation -SMDK generate commands runds a wizard and builds a sample project is a subdirectory. Let's get started: +SMDK generate commands runs a wizard and builds a sample project in a subdirectory %copy first-line% ```bash diff --git a/data/cli-commands.yml b/data/cli-commands.yml index 3e122205..8d982093 100644 --- a/data/cli-commands.yml +++ b/data/cli-commands.yml @@ -33,6 +33,9 @@ cli-commands: - fluvio hub - fluvio hub download - fluvio hub list + - fluvio hub connector + - fluvio hub connector list + - fluvio hub connector download - fluvio profile - fluvio profile current @@ -94,3 +97,14 @@ cli-commands: - smdk publish - smdk set-public + - cdk + - cdk build + - cdk test + - cdk generate + - cdk deploy + - cdk deploy start + - cdk deploy shutdown + - cdk deploy list + - cdk deploy log + - cdk publish + - cdk set-public diff --git a/embeds/cli/help/cdk-build.md b/embeds/cli/help/cdk-build.md new file mode 100644 index 00000000..c44423ca --- /dev/null +++ b/embeds/cli/help/cdk-build.md @@ -0,0 +1,10 @@ +``` +Build the Connector in the current working directory +Usage: cdk build [OPTIONS] [-- ...] +Arguments: + [EXTRA_ARGUMENTS]... Extra arguments to be passed to cargo +Options: + --release Release profile name [default: release] + -p, --package-name Optional package/project name + -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/cdk-deploy-list.md b/embeds/cli/help/cdk-deploy-list.md new file mode 100644 index 00000000..34037e08 --- /dev/null +++ b/embeds/cli/help/cdk-deploy-list.md @@ -0,0 +1,6 @@ +``` +Print the list of all deployed connectors +Usage: cdk deploy list +Options: + -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/cdk-deploy-log.md b/embeds/cli/help/cdk-deploy-log.md new file mode 100644 index 00000000..4633a019 --- /dev/null +++ b/embeds/cli/help/cdk-deploy-log.md @@ -0,0 +1,5 @@ +``` +Print the connector's logs +Usage: cdk deploy log Options: +Arguments: -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/cdk-deploy-shutdown.md b/embeds/cli/help/cdk-deploy-shutdown.md new file mode 100644 index 00000000..d7e31a09 --- /dev/null +++ b/embeds/cli/help/cdk-deploy-shutdown.md @@ -0,0 +1,8 @@ +``` +Shutdown the Connector's deployment +Usage: cdk deploy shutdown +Arguments: + +Options: + -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/cdk-deploy-start.md b/embeds/cli/help/cdk-deploy-start.md new file mode 100644 index 00000000..5356e0f1 --- /dev/null +++ b/embeds/cli/help/cdk-deploy-start.md @@ -0,0 +1,10 @@ +``` +Start new deployment for the given connector config +Usage: cdk deploy start [OPTIONS] --config +Options: + -c, --config Path to configuration file in YAML format + -s, --secrets Path to file with secrets. Secrets are 'key=value' pairs separated by the + new line character. Optional + --ipkg Deploy from local package file + -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/cdk-deploy.md b/embeds/cli/help/cdk-deploy.md new file mode 100644 index 00000000..eaa6caa3 --- /dev/null +++ b/embeds/cli/help/cdk-deploy.md @@ -0,0 +1,16 @@ +``` +Deploy the Connector from the current working directory +Usage: cdk deploy [OPTIONS] [-- ...] +Commands: + start Start new deployment for the given connector config + shutdown Shutdown the Connector's deployment + list Print the list of all deployed connectors + log Print the connector's logs + help Print this message or the help of the given subcommand(s) +Arguments: + [EXTRA_ARGUMENTS]... Extra arguments to be passed to cargo +Options: + --release Release profile name [default: release] + -p, --package-name Optional package/project name + -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/cdk-generate.md b/embeds/cli/help/cdk-generate.md new file mode 100644 index 00000000..caa84c78 --- /dev/null +++ b/embeds/cli/help/cdk-generate.md @@ -0,0 +1,10 @@ +``` +Generate new SmartConnector project +Usage: cdk generate [OPTIONS] [NAME] +Arguments: + [NAME] SmartConnector Project Name +Options: + --destination Local path to generate the SmartConnector project. Default to directory + with project name, created in current directory [env: CDK_DESTINATION=] + -h, --help Print help (see more with '--help') +``` \ No newline at end of file diff --git a/embeds/cli/help/cdk-publish.md b/embeds/cli/help/cdk-publish.md new file mode 100644 index 00000000..3f413393 --- /dev/null +++ b/embeds/cli/help/cdk-publish.md @@ -0,0 +1,11 @@ +``` +Publish Connector package to the Hub +Usage: cdk publish [OPTIONS] [PACKAGE_META] +Arguments: + [PACKAGE_META] +Options: + --public-yes don't ask for confirmation of public package publish + --target provide target platform for the package. Optional. By default the host's + one is used [default: aarch64-apple-darwin] + -h, --help Print help (see more with '--help') +``` \ No newline at end of file diff --git a/embeds/cli/help/cdk-set-public.md b/embeds/cli/help/cdk-set-public.md new file mode 100644 index 00000000..252d93f3 --- /dev/null +++ b/embeds/cli/help/cdk-set-public.md @@ -0,0 +1,4 @@ +``` +Set connector visibility to public Options: +Usage: cdk set-public -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/cdk-test.md b/embeds/cli/help/cdk-test.md new file mode 100644 index 00000000..4bedd451 --- /dev/null +++ b/embeds/cli/help/cdk-test.md @@ -0,0 +1,14 @@ +``` +Build and run the Connector in the current working directory +Usage: cdk test [OPTIONS] [-- ...] +Arguments: + [EXTRA_ARGUMENTS]... Extra arguments to be passed to cargo +Options: + --release Release profile name [default: release] + -p, --package-name Optional package/project name + -c, --config Path to configuration file in YAML format [default: + sample-config.yaml] + -s, --secrets Path to file with secrets. Secrets are 'key=value' pairs + separated by the new line character. Optional + -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/cdk.md b/embeds/cli/help/cdk.md new file mode 100644 index 00000000..54bfe97f --- /dev/null +++ b/embeds/cli/help/cdk.md @@ -0,0 +1,14 @@ +``` +Connector Development Kit +Usage: cdk +Commands: + build Build the Connector in the current working directory + test Build and run the Connector in the current working directory + generate Generate new SmartConnector project + deploy Deploy the Connector from the current working directory + publish Publish Connector package to the Hub + set-public Set connector visibility to public + help Print this message or the help of the given subcommand(s) +Options: + -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cloud-cluster-create.md b/embeds/cli/help/fluvio-cloud-cluster-create.md index b01051f2..66eb5d4f 100644 --- a/embeds/cli/help/fluvio-cloud-cluster-create.md +++ b/embeds/cli/help/fluvio-cloud-cluster-create.md @@ -1,10 +1,8 @@ ``` fluvio-cloud-cluster-create Create a new Fluvio cluster - USAGE: fluvio-cloud cluster create [OPTIONS] - OPTIONS: -h, --help Print help information --profile The name of the Profile to save diff --git a/embeds/cli/help/fluvio-cloud-cluster-delete.md b/embeds/cli/help/fluvio-cloud-cluster-delete.md index 2fe5a473..e1704be8 100644 --- a/embeds/cli/help/fluvio-cloud-cluster-delete.md +++ b/embeds/cli/help/fluvio-cloud-cluster-delete.md @@ -1,13 +1,10 @@ ``` fluvio-cloud-cluster-delete Delete an existing Fluvio cluster - USAGE: fluvio-cloud cluster delete - ARGS: The email for the cluster to be deleted - OPTIONS: -h, --help Print help information ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cloud-cluster-list.md b/embeds/cli/help/fluvio-cloud-cluster-list.md index eff63228..4951aff6 100644 --- a/embeds/cli/help/fluvio-cloud-cluster-list.md +++ b/embeds/cli/help/fluvio-cloud-cluster-list.md @@ -1,10 +1,8 @@ ``` fluvio-cloud-cluster-list List all Fluvio clusters - USAGE: fluvio-cloud cluster list - OPTIONS: -h, --help Print help information ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cloud-cluster.md b/embeds/cli/help/fluvio-cloud-cluster.md index 595762f5..cf0bb150 100644 --- a/embeds/cli/help/fluvio-cloud-cluster.md +++ b/embeds/cli/help/fluvio-cloud-cluster.md @@ -1,13 +1,10 @@ ``` fluvio-cloud-cluster View Cluster information - USAGE: fluvio-cloud cluster - OPTIONS: -h, --help Print help information - SUBCOMMANDS: create Create a new Fluvio cluster delete Delete an existing Fluvio cluster diff --git a/embeds/cli/help/fluvio-cloud-connector-config.md b/embeds/cli/help/fluvio-cloud-connector-config.md index 652e33f8..3a858a36 100644 --- a/embeds/cli/help/fluvio-cloud-connector-config.md +++ b/embeds/cli/help/fluvio-cloud-connector-config.md @@ -1,13 +1,10 @@ ``` fluvio-cloud-connector-config Show the connector configuration details - USAGE: - fluvio-cloud connector config - + fluvio-cloud connector config [OPTIONS] ARGS: Name of connector - OPTIONS: -h, --help Print help information ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cloud-connector-create.md b/embeds/cli/help/fluvio-cloud-connector-create.md index eb27f25b..33d32272 100644 --- a/embeds/cli/help/fluvio-cloud-connector-create.md +++ b/embeds/cli/help/fluvio-cloud-connector-create.md @@ -1,10 +1,8 @@ ``` fluvio-cloud-connector-create Create a new Connector - USAGE: - fluvio-cloud connector create --config - + fluvio-cloud connector create [OPTIONS] --config OPTIONS: -c, --config Name of connector -h, --help Print help information diff --git a/embeds/cli/help/fluvio-cloud-connector-delete.md b/embeds/cli/help/fluvio-cloud-connector-delete.md index 3aeb3471..fe39e84b 100644 --- a/embeds/cli/help/fluvio-cloud-connector-delete.md +++ b/embeds/cli/help/fluvio-cloud-connector-delete.md @@ -1,13 +1,10 @@ ``` fluvio-cloud-connector-delete Delete a Connector - USAGE: - fluvio-cloud connector delete ... - + fluvio-cloud connector delete [OPTIONS] ... ARGS: ... One or more name(s) of the connector(s) to be deleted - OPTIONS: -h, --help Print help information ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cloud-connector-list.md b/embeds/cli/help/fluvio-cloud-connector-list.md index 6663d653..68ee9ce7 100644 --- a/embeds/cli/help/fluvio-cloud-connector-list.md +++ b/embeds/cli/help/fluvio-cloud-connector-list.md @@ -1,10 +1,8 @@ ``` fluvio-cloud-connector-list List all Connectors - USAGE: - fluvio-cloud connector list - + fluvio-cloud connector list [OPTIONS] OPTIONS: -h, --help Print help information ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cloud-connector-logs.md b/embeds/cli/help/fluvio-cloud-connector-logs.md index ed6c0572..81b40f19 100644 --- a/embeds/cli/help/fluvio-cloud-connector-logs.md +++ b/embeds/cli/help/fluvio-cloud-connector-logs.md @@ -1,13 +1,10 @@ ``` fluvio-cloud-connector-logs View connector logs - USAGE: - fluvio-cloud connector logs - + fluvio-cloud connector logs [OPTIONS] ARGS: Name of connector - OPTIONS: -h, --help Print help information ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cloud-connector-update.md b/embeds/cli/help/fluvio-cloud-connector-update.md index 9af40a50..9096fdb4 100644 --- a/embeds/cli/help/fluvio-cloud-connector-update.md +++ b/embeds/cli/help/fluvio-cloud-connector-update.md @@ -1,10 +1,8 @@ ``` fluvio-cloud-connector-update Update and restart a Connector - USAGE: - fluvio-cloud connector update --config - + fluvio-cloud connector update [OPTIONS] --config OPTIONS: -c, --config Name of connector -h, --help Print help information diff --git a/embeds/cli/help/fluvio-cloud-connector.md b/embeds/cli/help/fluvio-cloud-connector.md index 7d3885b2..2bf7e3a7 100644 --- a/embeds/cli/help/fluvio-cloud-connector.md +++ b/embeds/cli/help/fluvio-cloud-connector.md @@ -1,13 +1,10 @@ ``` fluvio-cloud-connector View Fluvio Connector information - USAGE: fluvio-cloud connector - OPTIONS: -h, --help Print help information - SUBCOMMANDS: config Show the connector configuration details create Create a new Connector diff --git a/embeds/cli/help/fluvio-cloud-login.md b/embeds/cli/help/fluvio-cloud-login.md index 8060f9e7..7edf1b06 100644 --- a/embeds/cli/help/fluvio-cloud-login.md +++ b/embeds/cli/help/fluvio-cloud-login.md @@ -1,10 +1,8 @@ ``` fluvio-cloud-login Log into Infinyon Cloud with Oauth2 or username/password - USAGE: fluvio-cloud login [OPTIONS] - OPTIONS: --email Infinyon Cloud email to use for logging in -h, --help Print help information diff --git a/embeds/cli/help/fluvio-cloud-logout.md b/embeds/cli/help/fluvio-cloud-logout.md index 61a81220..e8841597 100644 --- a/embeds/cli/help/fluvio-cloud-logout.md +++ b/embeds/cli/help/fluvio-cloud-logout.md @@ -1,10 +1,8 @@ ``` fluvio-cloud-logout Log out of a Infinyon Cloud account - USAGE: fluvio-cloud logout - OPTIONS: -h, --help Print help information ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cloud-usage.md b/embeds/cli/help/fluvio-cloud-usage.md index 54ce0906..a236af08 100644 --- a/embeds/cli/help/fluvio-cloud-usage.md +++ b/embeds/cli/help/fluvio-cloud-usage.md @@ -1,10 +1,8 @@ ``` fluvio-cloud-usage Print cluster stats - USAGE: fluvio-cloud usage - OPTIONS: -h, --help Print help information ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cloud.md b/embeds/cli/help/fluvio-cloud.md index 9f8a3f67..5cda4b2b 100644 --- a/embeds/cli/help/fluvio-cloud.md +++ b/embeds/cli/help/fluvio-cloud.md @@ -1,12 +1,9 @@ ``` fluvio-cloud - USAGE: fluvio-cloud - OPTIONS: -h, --help Print help information - SUBCOMMANDS: cluster View Cluster information connector View Fluvio Connector information diff --git a/embeds/cli/help/fluvio-cluster-check.md b/embeds/cli/help/fluvio-cluster-check.md index 1df80df0..b834e0dd 100644 --- a/embeds/cli/help/fluvio-cluster-check.md +++ b/embeds/cli/help/fluvio-cluster-check.md @@ -1,9 +1,7 @@ ``` Check that all requirements for cluster startup are met - Usage: fluvio cluster check [OPTIONS] - Options: --fix Attempt to fix recoverable errors - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-delete.md b/embeds/cli/help/fluvio-cluster-delete.md index c0d2c4f0..60c9495a 100644 --- a/embeds/cli/help/fluvio-cluster-delete.md +++ b/embeds/cli/help/fluvio-cluster-delete.md @@ -1,12 +1,10 @@ ``` Uninstall a Fluvio cluster - Usage: fluvio cluster delete [OPTIONS] - Options: --namespace --local Remove only local spu/sc(custom) fluvio installation --k8 Remove only k8 fluvio installation --sys delete system chart - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-diagnostics.md b/embeds/cli/help/fluvio-cluster-diagnostics.md index 5832931b..b7ecdd3b 100644 --- a/embeds/cli/help/fluvio-cluster-diagnostics.md +++ b/embeds/cli/help/fluvio-cluster-diagnostics.md @@ -1,12 +1,10 @@ ``` Collect anonymous diagnostic information to help with debugging - Usage: fluvio cluster diagnostics [OPTIONS] - Options: --quiet --k8 --local --cloud - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-spg-create.md b/embeds/cli/help/fluvio-cluster-spg-create.md index 5e1198bd..f6540aaf 100644 --- a/embeds/cli/help/fluvio-cluster-spg-create.md +++ b/embeds/cli/help/fluvio-cluster-spg-create.md @@ -1,15 +1,12 @@ ``` Create a new managed SPU Group - fluvio cluster spg create [OPTIONS] - Arguments: The name for the new SPU Group - Options: -r, --replicas The number of SPUs to create in this SPG [default: 1] --min-id Minimum SPU ID [default: 1] --rack Rack name --storage-size The amount of storage to assign to this SPG - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-spg-delete.md b/embeds/cli/help/fluvio-cluster-spg-delete.md index a0946866..d0aae1f0 100644 --- a/embeds/cli/help/fluvio-cluster-spg-delete.md +++ b/embeds/cli/help/fluvio-cluster-spg-delete.md @@ -1,11 +1,8 @@ ``` Delete a managed SPU Group - fluvio cluster spg delete - Arguments: The name of the SPU Group to delete - Options: - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-spg-list.md b/embeds/cli/help/fluvio-cluster-spg-list.md index 8e2bcde4..759d1e21 100644 --- a/embeds/cli/help/fluvio-cluster-spg-list.md +++ b/embeds/cli/help/fluvio-cluster-spg-list.md @@ -1,9 +1,7 @@ ``` List all SPU Groups - fluvio cluster spg list [OPTIONS] - Options: -O, --output Output [default: table] [possible values: table, yaml, json] - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-spg.md b/embeds/cli/help/fluvio-cluster-spg.md index 82796d17..04c2e469 100644 --- a/embeds/cli/help/fluvio-cluster-spg.md +++ b/embeds/cli/help/fluvio-cluster-spg.md @@ -1,13 +1,10 @@ ``` Manage and view SPU Groups (SPGs) - Usage: fluvio cluster spg - Commands: create Create a new managed SPU Group delete Delete a managed SPU Group list List all SPU Groups - Options: - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-spu-list.md b/embeds/cli/help/fluvio-cluster-spu-list.md index 63cd07fa..6662aca4 100644 --- a/embeds/cli/help/fluvio-cluster-spu-list.md +++ b/embeds/cli/help/fluvio-cluster-spu-list.md @@ -1,10 +1,8 @@ ``` List all SPUs known by this cluster (managed AND custom) - fluvio cluster spu list [OPTIONS] - Options: --custom Whether to list only custom SPUs -O, --output Output [default: table] [possible values: table, yaml, json] - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-spu-register.md b/embeds/cli/help/fluvio-cluster-spu-register.md index 0fdec4c9..5036c204 100644 --- a/embeds/cli/help/fluvio-cluster-spu-register.md +++ b/embeds/cli/help/fluvio-cluster-spu-register.md @@ -1,13 +1,11 @@ ``` Register a new custom SPU with the cluster - fluvio cluster spu register [OPTIONS] --id --public-server --private-server - Options: -i, --id SPU id -n, --name SPU name -r, --rack Rack name -p, --public-server Public server::port -v, --private-server Private server::port - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-spu-unregister.md b/embeds/cli/help/fluvio-cluster-spu-unregister.md index ba346c01..925c2260 100644 --- a/embeds/cli/help/fluvio-cluster-spu-unregister.md +++ b/embeds/cli/help/fluvio-cluster-spu-unregister.md @@ -1,10 +1,8 @@ ``` Unregister a custom SPU from the cluster - fluvio cluster spu unregister [OPTIONS] - Options: -i, --id SPU id -n, --name SPU name - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-spu.md b/embeds/cli/help/fluvio-cluster-spu.md index 0f372024..de5e3b27 100644 --- a/embeds/cli/help/fluvio-cluster-spu.md +++ b/embeds/cli/help/fluvio-cluster-spu.md @@ -1,13 +1,10 @@ ``` Manage and view Streaming Processing Units (SPUs) - Usage: fluvio cluster spu - Commands: register Register a new custom SPU with the cluster unregister Unregister a custom SPU from the cluster list List all SPUs known by this cluster (managed AND custom) - Options: - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-start.md b/embeds/cli/help/fluvio-cluster-start.md index e02e4cb1..c9ef8bb9 100644 --- a/embeds/cli/help/fluvio-cluster-start.md +++ b/embeds/cli/help/fluvio-cluster-start.md @@ -1,8 +1,6 @@ ``` Install Fluvio cluster - Usage: fluvio cluster start [OPTIONS] - Options: --develop use local image @@ -31,13 +29,12 @@ Options: --spu-storage-size set spu storage size [default: 10] --skip-profile-creation - --spu number of SPU [default: 1] --rust-log RUST_LOG options --log-dir - log dir [default: /usr/local/var/log/fluvio] + log dir [default: /tmp] --sys-only installing/upgrade sys only --local @@ -57,7 +54,6 @@ Options: --server-key TLS: path to server private key --authorization-config-map - --skip-checks Whether to skip pre-install checks, defaults to false --setup @@ -69,5 +65,5 @@ Options: --connector-prefix Connector Prefix -h, --help - Print help information + Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster-upgrade.md b/embeds/cli/help/fluvio-cluster-upgrade.md index dc923ce1..4efe1d1e 100644 --- a/embeds/cli/help/fluvio-cluster-upgrade.md +++ b/embeds/cli/help/fluvio-cluster-upgrade.md @@ -1,8 +1,6 @@ ``` Upgrades an already-started Fluvio cluster - Usage: fluvio cluster upgrade [OPTIONS] - Options: --develop use local image @@ -31,13 +29,12 @@ Options: --spu-storage-size set spu storage size [default: 10] --skip-profile-creation - --spu number of SPU [default: 1] --rust-log RUST_LOG options --log-dir - log dir [default: /usr/local/var/log/fluvio] + log dir [default: /tmp] --sys-only installing/upgrade sys only --local @@ -57,7 +54,6 @@ Options: --server-key TLS: path to server private key --authorization-config-map - --skip-checks Whether to skip pre-install checks, defaults to false --setup @@ -69,5 +65,5 @@ Options: --connector-prefix Connector Prefix -h, --help - Print help information + Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-cluster.md b/embeds/cli/help/fluvio-cluster.md index 46ada1ff..4be24d42 100644 --- a/embeds/cli/help/fluvio-cluster.md +++ b/embeds/cli/help/fluvio-cluster.md @@ -1,8 +1,6 @@ ``` Install or uninstall Fluvio cluster - Usage: fluvio cluster - Commands: start Install Fluvio cluster upgrade Upgrades an already-started Fluvio cluster @@ -11,7 +9,8 @@ Commands: spu Manage and view Streaming Processing Units (SPUs) spg Manage and view SPU Groups (SPGs) diagnostics Collect anonymous diagnostic information to help with debugging - + status Check the status of a Fluvio cluster + shutdown Shutdown cluster processes without deleting data Options: - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-completions.md b/embeds/cli/help/fluvio-completions.md index 33947a54..d95ddcd8 100644 --- a/embeds/cli/help/fluvio-completions.md +++ b/embeds/cli/help/fluvio-completions.md @@ -1,13 +1,10 @@ ``` Generate command-line completions for Fluvio - Usage: fluvio completions - Commands: bash Generate CLI completions for bash zsh Generate CLI completions for zsh fish Generate CLI completions for fish - Options: - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-consume.md b/embeds/cli/help/fluvio-consume.md index 52e5569e..47fe6cd0 100644 --- a/embeds/cli/help/fluvio-consume.md +++ b/embeds/cli/help/fluvio-consume.md @@ -1,11 +1,8 @@ ``` Read messages from a topic/partition - Usage: fluvio consume [OPTIONS] - Arguments: Topic name - Options: -p, --partition Partition id [default: 0] @@ -38,9 +35,9 @@ Options: -O, --output Output [possible values: dynamic, text, binary, json, raw, table, full-table] --smartmodule - Name of the smart module + Name of the smartmodule --smartmodule-path - Path to the stmart module + Path to the smart module --aggregate-initial (Optional) Path to a file to use as an initial accumulator value with --aggregate -e, --params @@ -52,5 +49,5 @@ Options: -t, --transform (Optional) Transformation specification as JSON formatted string. E.g. fluvio consume topic-name --transform='{"uses":"infinyon/jolt@0.1.0","with":{"spec":"[{\"operation\":\"default\",\"spec\":{\"source\":\"test\"}}]"}}' -h, --help - Print help information (use `--help` for more detail) + Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-derived-stream-create.md b/embeds/cli/help/fluvio-derived-stream-create.md index 1340d293..f1c3fffb 100644 --- a/embeds/cli/help/fluvio-derived-stream-create.md +++ b/embeds/cli/help/fluvio-derived-stream-create.md @@ -1,9 +1,3 @@ ``` -Create a new SmartModule with a given name - -Usage: fluvio derived-stream create --config - -Options: - -c, --config The name for the new Managed Connector - -h, --help Print help information +Unable to find plugin 'fluvio-derived-stream'. Make sure it is installed in "/Users/telant/.fluvio/extensions-latest". ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-derived-stream-delete.md b/embeds/cli/help/fluvio-derived-stream-delete.md index 4cf97c90..f1c3fffb 100644 --- a/embeds/cli/help/fluvio-derived-stream-delete.md +++ b/embeds/cli/help/fluvio-derived-stream-delete.md @@ -1,9 +1,3 @@ ``` -Usage: fluvio derived-stream delete - -Arguments: - - -Options: - -h, --help Print help information +Unable to find plugin 'fluvio-derived-stream'. Make sure it is installed in "/Users/telant/.fluvio/extensions-latest". ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-derived-stream-list.md b/embeds/cli/help/fluvio-derived-stream-list.md index dd1c554b..f1c3fffb 100644 --- a/embeds/cli/help/fluvio-derived-stream-list.md +++ b/embeds/cli/help/fluvio-derived-stream-list.md @@ -1,9 +1,3 @@ ``` -List all existing SmartModules - -Usage: fluvio derived-stream list [OPTIONS] - -Options: - -O, --output Output [default: table] [possible values: table, yaml, json] - -h, --help Print help information +Unable to find plugin 'fluvio-derived-stream'. Make sure it is installed in "/Users/telant/.fluvio/extensions-latest". ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-derived-stream.md b/embeds/cli/help/fluvio-derived-stream.md index 63a5800e..f1c3fffb 100644 --- a/embeds/cli/help/fluvio-derived-stream.md +++ b/embeds/cli/help/fluvio-derived-stream.md @@ -1,13 +1,3 @@ ``` -Create and manage DerivedStreams - -Usage: fluvio derived-stream - -Commands: - create Create a new SmartModule with a given name - list List all existing SmartModules - delete - -Options: - -h, --help Print help information (use `--help` for more detail) +Unable to find plugin 'fluvio-derived-stream'. Make sure it is installed in "/Users/telant/.fluvio/extensions-latest". ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-hub-connector-download.md b/embeds/cli/help/fluvio-hub-connector-download.md new file mode 100644 index 00000000..f9b28fb1 --- /dev/null +++ b/embeds/cli/help/fluvio-hub-connector-download.md @@ -0,0 +1,10 @@ +``` +Download SmartConnector to the local folder +Usage: fluvio hub connector download [OPTIONS] +Arguments: + SmartConnector name: e.g. infinyon/salesforce-sink@v0.0.1 +Options: + -o, --output Target local folder or file name + --target Target platform for the package. Optional. By default the host's one is used [default: aarch64-apple-darwin] + -h, --help Print help (see more with '--help') +``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-hub-connector-list.md b/embeds/cli/help/fluvio-hub-connector-list.md new file mode 100644 index 00000000..39f539d9 --- /dev/null +++ b/embeds/cli/help/fluvio-hub-connector-list.md @@ -0,0 +1,7 @@ +``` +List all available SmartConnectors +Usage: fluvio hub connector list [OPTIONS] +Options: + -O, --output Output [default: table] [possible values: table, yaml, json] + -h, --help Print help (see more with '--help') +``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-hub-connector.md b/embeds/cli/help/fluvio-hub-connector.md new file mode 100644 index 00000000..9b5fc53b --- /dev/null +++ b/embeds/cli/help/fluvio-hub-connector.md @@ -0,0 +1,9 @@ +``` +List available Connectors in the hub +Usage: fluvio hub connector +Commands: + list List all available SmartConnectors + download Download SmartConnector to the local folder +Options: + -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-hub-download.md b/embeds/cli/help/fluvio-hub-download.md index 5f87b9e0..f52af6c2 100644 --- a/embeds/cli/help/fluvio-hub-download.md +++ b/embeds/cli/help/fluvio-hub-download.md @@ -1,11 +1,8 @@ ``` Download a SmartModule from the hub - Usage: fluvio hub download [OPTIONS] - Arguments: SmartModule name: e.g. infinyon/jolt@v0.0.1 - Options: -c, --cluster Address of cluster --tls Enable TLS @@ -17,5 +14,5 @@ Options: -P, --profile --local just download package to local filesystem --ipkg given local package file, download to cluster - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-hub-list.md b/embeds/cli/help/fluvio-hub-list.md index 8a4c4ed7..f0d14d31 100644 --- a/embeds/cli/help/fluvio-hub-list.md +++ b/embeds/cli/help/fluvio-hub-list.md @@ -1,9 +1,7 @@ ``` List available SmartModules in the hub - Usage: fluvio hub list [OPTIONS] - Options: -O, --output Output [default: table] [possible values: table, yaml, json] - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-hub.md b/embeds/cli/help/fluvio-hub.md index 14951194..cc9fa5c9 100644 --- a/embeds/cli/help/fluvio-hub.md +++ b/embeds/cli/help/fluvio-hub.md @@ -1,12 +1,10 @@ ``` Work with the SmartModule Hub - Usage: fluvio hub - Commands: - download Download a SmartModule from the hub - list List available SmartModules in the hub - + download Download a SmartModule from the hub + list List available SmartModules in the hub + connector List available Connectors in the hub Options: - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-install.md b/embeds/cli/help/fluvio-install.md index dfd25539..8d4aa3c9 100644 --- a/embeds/cli/help/fluvio-install.md +++ b/embeds/cli/help/fluvio-install.md @@ -1,12 +1,9 @@ ``` Install Fluvio plugins - -Usage: fluvio install [OPTIONS] - +Usage: fluvio install [OPTIONS] [PACKAGE] Arguments: - The ID of a package to install, e.g. "fluvio/fluvio-cloud" - + [PACKAGE] The ID of a package to install, e.g. "fluvio/fluvio-cloud" Options: --develop Install the latest prerelease rather than the latest release - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-partition-list.md b/embeds/cli/help/fluvio-partition-list.md index 368075c1..2b5a7ea9 100644 --- a/embeds/cli/help/fluvio-partition-list.md +++ b/embeds/cli/help/fluvio-partition-list.md @@ -1,9 +1,7 @@ ``` List all of the Partitions in this cluster - fluvio partition list [OPTIONS] - Options: -O, --output Output [default: table] [possible values: table, yaml, json] - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-partition.md b/embeds/cli/help/fluvio-partition.md index 5a8c3308..7c6cf27d 100644 --- a/embeds/cli/help/fluvio-partition.md +++ b/embeds/cli/help/fluvio-partition.md @@ -1,11 +1,8 @@ ``` Manage and view Partitions - Usage: fluvio partition - Commands: list List all of the Partitions in this cluster - Options: - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-produce.md b/embeds/cli/help/fluvio-produce.md index 85f307a5..4f10ea75 100644 --- a/embeds/cli/help/fluvio-produce.md +++ b/embeds/cli/help/fluvio-produce.md @@ -1,11 +1,8 @@ ``` Write messages to a topic/partition - Usage: fluvio produce [OPTIONS] - Arguments: The name of the Topic to produce to - Options: -v, --verbose Print progress output when sending records @@ -28,5 +25,5 @@ Options: --delivery-semantic Delivery guarantees that producer must respect. Supported values: at_most_once (AtMostOnce) - send records without waiting from response, at_least_once (AtLeastOnce) - send records and retry if error occurred [default: at-least-once] -h, --help - Print help information + Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-profile-current.md b/embeds/cli/help/fluvio-profile-current.md index 3bec3dd9..1f92f1cd 100644 --- a/embeds/cli/help/fluvio-profile-current.md +++ b/embeds/cli/help/fluvio-profile-current.md @@ -1,8 +1,4 @@ ``` -Print the name of the current context - -Usage: fluvio profile current - -Options: - -h, --help Print help information +Print the name of the current context Options: +Usage: fluvio profile current -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-profile-delete-cluster.md b/embeds/cli/help/fluvio-profile-delete-cluster.md index f210fca2..ce6b29e6 100644 --- a/embeds/cli/help/fluvio-profile-delete-cluster.md +++ b/embeds/cli/help/fluvio-profile-delete-cluster.md @@ -1,12 +1,9 @@ ``` Delete the named cluster - Usage: fluvio profile delete-cluster [OPTIONS] - Arguments: The name of a cluster connection to delete - Options: -f, --force Deletes a cluster even if its active - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-profile-delete.md b/embeds/cli/help/fluvio-profile-delete.md index 076f962f..e3a226c3 100644 --- a/embeds/cli/help/fluvio-profile-delete.md +++ b/embeds/cli/help/fluvio-profile-delete.md @@ -1,11 +1,8 @@ ``` Delete the named profile - Usage: fluvio profile delete - Arguments: - Options: - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-profile-export.md b/embeds/cli/help/fluvio-profile-export.md index cce21704..a53ef809 100644 --- a/embeds/cli/help/fluvio-profile-export.md +++ b/embeds/cli/help/fluvio-profile-export.md @@ -1,12 +1,9 @@ ``` Export a profile for use in other applications - Usage: fluvio profile export [OPTIONS] [PROFILE_NAME] - Arguments: [PROFILE_NAME] - Options: -O, --output [default: json] [possible values: table, yaml, json] - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-profile-list.md b/embeds/cli/help/fluvio-profile-list.md index 42f8ceb3..90f6e829 100644 --- a/embeds/cli/help/fluvio-profile-list.md +++ b/embeds/cli/help/fluvio-profile-list.md @@ -1,9 +1,7 @@ ``` Display the entire Fluvio configuration - Usage: fluvio profile list [OPTIONS] - Options: -O, --output Output [default: table] [possible values: table, yaml, json] - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-profile-rename.md b/embeds/cli/help/fluvio-profile-rename.md index d1356799..30c55c34 100644 --- a/embeds/cli/help/fluvio-profile-rename.md +++ b/embeds/cli/help/fluvio-profile-rename.md @@ -1,12 +1,9 @@ ``` Rename a profile - Usage: fluvio profile rename - Arguments: The name of the profile to rename The new name to give the profile - Options: - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-profile-switch.md b/embeds/cli/help/fluvio-profile-switch.md index 00893956..09c5c617 100644 --- a/embeds/cli/help/fluvio-profile-switch.md +++ b/embeds/cli/help/fluvio-profile-switch.md @@ -1,11 +1,8 @@ ``` Switch to the named profile - Usage: fluvio profile switch - Arguments: - Options: - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-profile-sync.md b/embeds/cli/help/fluvio-profile-sync.md index 1099c99d..b7d4a678 100644 --- a/embeds/cli/help/fluvio-profile-sync.md +++ b/embeds/cli/help/fluvio-profile-sync.md @@ -1,12 +1,9 @@ ``` Sync a profile from a cluster - fluvio profile sync - Commands: k8 Sync a profile from a Kubernetes cluster local Sync a profile from a local cluster - Options: - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-profile.md b/embeds/cli/help/fluvio-profile.md index 131e38b6..e41acec1 100644 --- a/embeds/cli/help/fluvio-profile.md +++ b/embeds/cli/help/fluvio-profile.md @@ -1,8 +1,6 @@ ``` Manage Profiles, which describe linked clusters - Usage: fluvio profile [COMMAND] - Commands: current Print the name of the current context delete Delete the named profile @@ -12,7 +10,6 @@ Commands: switch Switch to the named profile sync Sync a profile from a cluster export Export a profile for use in other applications - Options: - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-run-metadata.md b/embeds/cli/help/fluvio-run-metadata.md index dd373fab..5972e267 100644 --- a/embeds/cli/help/fluvio-run-metadata.md +++ b/embeds/cli/help/fluvio-run-metadata.md @@ -1,8 +1,4 @@ ``` -Return plugin metadata as JSON - -Usage: fluvio-run metadata - -Options: - -h, --help Print help information +Return plugin metadata as JSON Options: +Usage: fluvio-run metadata -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-run-sc.md b/embeds/cli/help/fluvio-run-sc.md index e005ce01..d7160f5e 100644 --- a/embeds/cli/help/fluvio-run-sc.md +++ b/embeds/cli/help/fluvio-run-sc.md @@ -1,8 +1,6 @@ ``` Run a new Streaming Controller (SC) - Usage: fluvio-run sc [OPTIONS] - Options: --local running in local mode only @@ -11,7 +9,6 @@ Options: --bind-private Address for internal service -n, --namespace - --tls enable tls --server-cert @@ -33,5 +30,5 @@ Options: --white-list only allow white list of controllers -h, --help - Print help information + Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-run-spu.md b/embeds/cli/help/fluvio-run-spu.md index 38246175..52d6db8b 100644 --- a/embeds/cli/help/fluvio-run-spu.md +++ b/embeds/cli/help/fluvio-run-spu.md @@ -1,8 +1,6 @@ ``` Run a new Streaming Processing Unit (SPU) - Usage: fluvio-run spu [OPTIONS] - Options: -i, --id SPU unique identifier @@ -35,5 +33,5 @@ Options: --bind-non-tls-public TLS: address of non tls public service, required -h, --help - Print help information + Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-run-version.md b/embeds/cli/help/fluvio-run-version.md index 5fd98705..1f90188c 100644 --- a/embeds/cli/help/fluvio-run-version.md +++ b/embeds/cli/help/fluvio-run-version.md @@ -1,8 +1,4 @@ ``` -Print version information - -Usage: fluvio-run version - -Options: - -h, --help Print help information +Print version information Options: +Usage: fluvio-run version -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-run.md b/embeds/cli/help/fluvio-run.md index e455a244..3bdc4e2f 100644 --- a/embeds/cli/help/fluvio-run.md +++ b/embeds/cli/help/fluvio-run.md @@ -1,14 +1,12 @@ ``` Usage: fluvio-run - Commands: spu Run a new Streaming Processing Unit (SPU) sc Run a new Streaming Controller (SC) metadata Return plugin metadata as JSON version Print version information help Print this message or the help of the given subcommand(s) - Options: - -h, --help Print help information - -V, --version Print version information + -h, --help Print help + -V, --version Print version ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-smartmodule-create.md b/embeds/cli/help/fluvio-smartmodule-create.md index 5cda8439..75548f6a 100644 --- a/embeds/cli/help/fluvio-smartmodule-create.md +++ b/embeds/cli/help/fluvio-smartmodule-create.md @@ -1,13 +1,10 @@ ``` Create a new SmartModule with a given name - Usage: fluvio smartmodule create [OPTIONS] --wasm-file - Arguments: The name of the SmartModule to create - Options: --wasm-file The path to a WASM binary to create the SmartModule from --package The path to the SmartModule package (experimental) - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-smartmodule-delete.md b/embeds/cli/help/fluvio-smartmodule-delete.md index 15c06572..e31e21ba 100644 --- a/embeds/cli/help/fluvio-smartmodule-delete.md +++ b/embeds/cli/help/fluvio-smartmodule-delete.md @@ -1,12 +1,9 @@ ``` Delete one or more SmartModules with the given name(s) - Usage: fluvio smartmodule delete [OPTIONS] ... - Arguments: - ... One or more name(s) of the smart module(s) to be deleted - + ... One or more name(s) of the smartmodule(s) to be deleted Options: -c, --continue-on-error Continue deleting in case of an error - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-smartmodule-list.md b/embeds/cli/help/fluvio-smartmodule-list.md index 9f3eacd3..9b0209c2 100644 --- a/embeds/cli/help/fluvio-smartmodule-list.md +++ b/embeds/cli/help/fluvio-smartmodule-list.md @@ -1,10 +1,8 @@ ``` List all existing SmartModules - Usage: fluvio smartmodule list [OPTIONS] - Options: -O, --output Output [default: table] [possible values: table, yaml, json] --filter - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-smartmodule-watch.md b/embeds/cli/help/fluvio-smartmodule-watch.md index 12e7e8e3..2d5cbcdc 100644 --- a/embeds/cli/help/fluvio-smartmodule-watch.md +++ b/embeds/cli/help/fluvio-smartmodule-watch.md @@ -1,9 +1,7 @@ ``` -List all existing SmartModules - +Watch for changes to SmartModules Usage: fluvio smartmodule watch [OPTIONS] - Options: -O, --output Output [default: table] [possible values: table, yaml, json] - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-smartmodule.md b/embeds/cli/help/fluvio-smartmodule.md index cf7a1313..792e7d57 100644 --- a/embeds/cli/help/fluvio-smartmodule.md +++ b/embeds/cli/help/fluvio-smartmodule.md @@ -1,14 +1,11 @@ ``` Create and manage SmartModules - Usage: fluvio smartmodule - Commands: create Create a new SmartModule with a given name list List all existing SmartModules - watch List all existing SmartModules + watch Watch for changes to SmartModules delete Delete one or more SmartModules with the given name(s) - Options: - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-table-format-create.md b/embeds/cli/help/fluvio-table-format-create.md index 2f18418f..eaf39252 100644 --- a/embeds/cli/help/fluvio-table-format-create.md +++ b/embeds/cli/help/fluvio-table-format-create.md @@ -1,9 +1,7 @@ ``` Create a new TableFormat display - fluvio table-format create --config - Options: -c, --config The path to the TableFormat config - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-table-format-delete.md b/embeds/cli/help/fluvio-table-format-delete.md index 4dccc05c..bf656cb0 100644 --- a/embeds/cli/help/fluvio-table-format-delete.md +++ b/embeds/cli/help/fluvio-table-format-delete.md @@ -1,11 +1,8 @@ ``` Delete a TableFormat display - fluvio table-format delete - Arguments: The name of the connector to delete - Options: - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-table-format-list.md b/embeds/cli/help/fluvio-table-format-list.md index 7c2769d0..9f1b26ec 100644 --- a/embeds/cli/help/fluvio-table-format-list.md +++ b/embeds/cli/help/fluvio-table-format-list.md @@ -1,9 +1,7 @@ ``` List all TableFormat display - fluvio table-format list [OPTIONS] - Options: -O, --output Output [default: table] [possible values: table, yaml, json] - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-table-format.md b/embeds/cli/help/fluvio-table-format.md index c1b969dd..2d768bba 100644 --- a/embeds/cli/help/fluvio-table-format.md +++ b/embeds/cli/help/fluvio-table-format.md @@ -1,13 +1,10 @@ ``` Create a TableFormat display specification - Usage: fluvio table-format - Commands: create Create a new TableFormat display delete Delete a TableFormat display list List all TableFormat display - Options: - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-topic-create.md b/embeds/cli/help/fluvio-topic-create.md index 9ddb2eba..8ef49670 100644 --- a/embeds/cli/help/fluvio-topic-create.md +++ b/embeds/cli/help/fluvio-topic-create.md @@ -1,11 +1,8 @@ ``` Create a Topic with the given name - fluvio topic create [OPTIONS] - Arguments: The name of the Topic to create - Options: -p, --partitions The number of Partitions to give the Topic [default: 1] -r, --replication The number of full replicas of the Topic to keep [default: 1] @@ -16,5 +13,5 @@ Options: --segment-size Segment size (by default measured in bytes) Ex: `2048`, '2 Ki', '10 MiB', `1 GB` --compression-type Compression configuration for topic --max-partition-size Max partition size (by default measured in bytes) Ex: `2048`, '2 Ki', '10 MiB', `1 GB` - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-topic-delete.md b/embeds/cli/help/fluvio-topic-delete.md index a007c3b8..be4e7593 100644 --- a/embeds/cli/help/fluvio-topic-delete.md +++ b/embeds/cli/help/fluvio-topic-delete.md @@ -1,12 +1,9 @@ ``` Delete one or more Topics with the given name(s) - fluvio topic delete [OPTIONS] ... - Arguments: ... One or more name(s) of the topic(s) to be deleted - Options: -c, --continue-on-error Continue deleting in case of an error - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-topic-describe.md b/embeds/cli/help/fluvio-topic-describe.md index a541f3fd..424adf60 100644 --- a/embeds/cli/help/fluvio-topic-describe.md +++ b/embeds/cli/help/fluvio-topic-describe.md @@ -1,12 +1,9 @@ ``` Print detailed information about a Topic - fluvio topic describe [OPTIONS] - Arguments: The name of the Topic to describe - Options: -O, --output Output [default: table] [possible values: table, yaml, json] - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-topic-list.md b/embeds/cli/help/fluvio-topic-list.md index f0557949..d7bb3200 100644 --- a/embeds/cli/help/fluvio-topic-list.md +++ b/embeds/cli/help/fluvio-topic-list.md @@ -1,9 +1,7 @@ ``` List all of the Topics in the cluster - fluvio topic list [OPTIONS] - Options: -O, --output Output [default: table] [possible values: table, yaml, json] - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-topic.md b/embeds/cli/help/fluvio-topic.md index af5c0472..fc68572c 100644 --- a/embeds/cli/help/fluvio-topic.md +++ b/embeds/cli/help/fluvio-topic.md @@ -1,14 +1,11 @@ ``` Manage and view Topics - Usage: fluvio topic - Commands: create Create a Topic with the given name delete Delete one or more Topics with the given name(s) describe Print detailed information about a Topic list List all of the Topics in the cluster - Options: - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-update.md b/embeds/cli/help/fluvio-update.md index bad6df32..0fbc1249 100644 --- a/embeds/cli/help/fluvio-update.md +++ b/embeds/cli/help/fluvio-update.md @@ -1,13 +1,10 @@ ``` Update the Fluvio CLI - Usage: fluvio update [OPTIONS] [PLUGINS]... - Arguments: [PLUGINS]... (Optional) the name of one or more plugins to update - Options: --develop Update to the latest prerelease rather than the latest release --dry-run Print output for update process but do not install updates - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio-version.md b/embeds/cli/help/fluvio-version.md index faf10ff4..e915f526 100644 --- a/embeds/cli/help/fluvio-version.md +++ b/embeds/cli/help/fluvio-version.md @@ -1,15 +1,12 @@ ``` Usage: fluvio-channel-cli [COMMAND] - Commands: create Create a local Fluvio release channel delete Delete a local Fluvio release channel list List local Fluvio release channels switch Change the active Fluvio release channel help Print this message or the help of the given subcommand(s) - Options: -h, --help - -h, --help Print help information - + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/fluvio.md b/embeds/cli/help/fluvio.md index cb814114..06caca38 100644 --- a/embeds/cli/help/fluvio.md +++ b/embeds/cli/help/fluvio.md @@ -1,26 +1,23 @@ ``` Fluvio Command Line Interface - fluvio-cli [OPTIONS] - Commands: - consume Read messages from a topic/partition - produce Write messages to a topic/partition - topic Manage and view Topics - partition Manage and view Partitions - smartmodule Create and manage SmartModules [aliases: sm] - table-format Create a TableFormat display specification [aliases: tf] - derived-stream Create and manage DerivedStreams [aliases: ds] - hub Work with the SmartModule Hub - profile Manage Profiles, which describe linked clusters - cluster Install or uninstall Fluvio cluster - install Install Fluvio plugins - update Update the Fluvio CLI - version Print Fluvio version information - completions Generate command-line completions for Fluvio - run Run Fluvio cluster components (SC and SPU) - cloud Cloud Operations - + consume Read messages from a topic/partition + produce Write messages to a topic/partition + topic Manage and view Topics + partition Manage and view Partitions + smartmodule Create and manage SmartModules [aliases: sm] + table-format Create a TableFormat display specification [aliases: tf] + hub Work with the SmartModule Hub + profile Manage Profiles, which describe linked clusters + cluster Install or uninstall Fluvio cluster + install Install Fluvio plugins + update Update the Fluvio CLI + version Print Fluvio version information + completions Generate command-line completions for Fluvio + run Run Fluvio cluster components (SC and SPU) + package Package publishing and management + cloud Cloud Operations Options: -c, --cluster Address of cluster --tls Enable TLS @@ -30,5 +27,5 @@ Options: --client-cert Path to TLS client certificate --client-key Path to TLS client private key -P, --profile - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/smdk-build.md b/embeds/cli/help/smdk-build.md index 73c67a0d..29bb15de 100644 --- a/embeds/cli/help/smdk-build.md +++ b/embeds/cli/help/smdk-build.md @@ -1,13 +1,10 @@ ``` Builds SmartModule into WASM - Usage: smdk build [OPTIONS] [-- ...] - Arguments: [EXTRA_ARGUMENTS]... Extra arguments to be passed to cargo - Options: --release Release profile name [default: release-lto] -p, --package-name Optional package/project name - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/smdk-generate.md b/embeds/cli/help/smdk-generate.md index 104574f6..15a8072f 100644 --- a/embeds/cli/help/smdk-generate.md +++ b/embeds/cli/help/smdk-generate.md @@ -1,23 +1,22 @@ ``` Generates a new SmartModule Project - Usage: smdk generate [OPTIONS] [NAME] - Arguments: [NAME] SmartModule Project Name - Options: --project-group SmartModule Project Group Name. Default to Hub ID, if set. Overrides Hub ID if provided [env: SMDK_PROJECT_GROUP=] --destination Local path to generate the SmartModule project. Default to directory with project name, created in current directory [env: SMDK_DESTINATION=] - --sm-type Type of SmartModule project to generate. Skip prompt if value given - [env: SMDK_SM_TYPE=] [possible values: filter, map, array-map, - aggregate, filter-map] + --sm-type Type of SmartModule project to generate. Skip prompt if value given [env: + SMDK_SM_TYPE=] [possible values: filter, map, array-map, aggregate, + filter-map] + --sm-public Visibility of SmartModule project to generate. Skip prompt if value + given [env: SMDK_SM_PUBLIC=] [possible values: true, false] --with-params Include SmartModule input parameters in generated SmartModule project. Skip prompt if value given [env: SMDK_WITH_PARAMS=] --no-params No SmartModule input parameters in generated SmartModule project. Skip prompt if value given [env: SMDK_NO_PARAMS=] - -h, --help Print help information (use `--help` for more detail) + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/smdk-load.md b/embeds/cli/help/smdk-load.md index 8d308b10..6ffeef5d 100644 --- a/embeds/cli/help/smdk-load.md +++ b/embeds/cli/help/smdk-load.md @@ -1,37 +1,21 @@ ``` Load SmartModule into Fluvio cluster - Usage: smdk load [OPTIONS] - Options: - --name - - --package-path - Optional path to SmartModule package directory - --release - Release profile name [default: release-lto] - -p, --package-name - Optional package/project name - --wasm-file - Optional wasm file path - -c, --cluster - Address of cluster - --tls - Enable TLS - --enable-client-cert - TLS: use client cert - --domain - Required if client cert is used - --ca-cert - Path to TLS ca cert, required when client cert is enabled - --client-cert - Path to TLS client certificate - --client-key - Path to TLS client private key - -P, --profile - - --dry-run - Validate package config files, and connection to cluster. Skip SmartModule load to cluster - -h, --help - Print help information + --name + --package-path Optional path to SmartModule package directory + --release Release profile name [default: release-lto] + -p, --package-name Optional package/project name + --wasm-file Optional wasm file path + -c, --cluster Address of cluster + --tls Enable TLS + --enable-client-cert TLS: use client cert + --domain Required if client cert is used + --ca-cert Path to TLS ca cert, required when client cert is enabled + --client-cert Path to TLS client certificate + --client-key Path to TLS client private key + -P, --profile + --dry-run Validate package config files, and connection to cluster. Skip + SmartModule load to cluster + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/smdk-publish.md b/embeds/cli/help/smdk-publish.md index fedb6d72..2d14ecc4 100644 --- a/embeds/cli/help/smdk-publish.md +++ b/embeds/cli/help/smdk-publish.md @@ -1,12 +1,9 @@ ``` Publish SmartModule to Hub - Usage: smdk publish [OPTIONS] [PACKAGE_META] - Arguments: [PACKAGE_META] - Options: - --pack do only the pack portion - -h, --help Print help information (use `--help` for more detail) + --public-yes don't ask for confirmation of public package publish + -h, --help Print help (see more with '--help') ``` \ No newline at end of file diff --git a/embeds/cli/help/smdk-set-public.md b/embeds/cli/help/smdk-set-public.md new file mode 100644 index 00000000..fa8a60f1 --- /dev/null +++ b/embeds/cli/help/smdk-set-public.md @@ -0,0 +1,4 @@ +``` +Set package as public Options: +Usage: smdk set-public -h, --help Print help +``` \ No newline at end of file diff --git a/embeds/cli/help/smdk-test.md b/embeds/cli/help/smdk-test.md index 6ac0484f..35d97ed5 100644 --- a/embeds/cli/help/smdk-test.md +++ b/embeds/cli/help/smdk-test.md @@ -1,13 +1,15 @@ ``` Test SmartModule - -Usage: smdk test [OPTIONS] - +Usage: smdk test [OPTIONS] [KEY] +Arguments: + [KEY] Key to use with the test record(s) Options: --text - + Provide test input with this flag --file - + Path to test file. Default: Read file line by line + --raw + Read the file as single record --release Release profile name [default: release-lto] -p, --package-name @@ -18,6 +20,12 @@ Options: (Optional) Extra input parameters passed to the smartmodule module. They should be passed using key=value format Eg. fluvio consume topic-name --filter filter.wasm -e foo=bar -e key=value -e one=1 + --transforms-file + (Optional) File path to transformation specification + -t, --transform + (Optional) Pass transformation specification as JSON formatted string. E.g. smdk test + --text '{}' + --transform='{"uses":"infinyon/jolt@0.1.0","with":{"spec":"[{\"operation\":\"default\",\"spec\":{\"source\":\"test\"}}]"}}' -h, --help - Print help information + Print help ``` \ No newline at end of file diff --git a/embeds/cli/help/smdk.md b/embeds/cli/help/smdk.md index 1c4390dc..8b3f41f4 100644 --- a/embeds/cli/help/smdk.md +++ b/embeds/cli/help/smdk.md @@ -1,16 +1,14 @@ ``` SmartModule Development Kit utility - Usage: smdk - Commands: - build Builds SmartModule into WASM - generate Generates a new SmartModule Project - test Test SmartModule - load Load SmartModule into Fluvio cluster - publish Publish SmartModule to Hub - help Print this message or the help of the given subcommand(s) - + build Builds SmartModule into WASM + generate Generates a new SmartModule Project + test Test SmartModule + load Load SmartModule into Fluvio cluster + publish Publish SmartModule to Hub + set-public Set package as public + help Print this message or the help of the given subcommand(s) Options: - -h, --help Print help information + -h, --help Print help ``` \ No newline at end of file diff --git a/embeds/client-examples/node/Dockerfile b/embeds/client-examples/node/Dockerfile index 9cf3d25b..ce165dc8 100644 --- a/embeds/client-examples/node/Dockerfile +++ b/embeds/client-examples/node/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18 +FROM node:16 # Run as the `fluvio` user instead of root ENV USER=fluvio diff --git a/embeds/client-examples/python/Dockerfile b/embeds/client-examples/python/Dockerfile index 9c384d2e..29dbfaab 100644 --- a/embeds/client-examples/python/Dockerfile +++ b/embeds/client-examples/python/Dockerfile @@ -9,11 +9,8 @@ WORKDIR /home/fluvio # Copy your Rust project ng Fluvio CLI for creating topics in example RUN curl -fsS https://packages.fluvio.io/v1/install.sh | bash -# Install Rust development environment to build Python wheel -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y - -# Add `fluvio` and `cargo` to PATH -ENV PATH="/home/fluvio/.fluvio/bin:/home/fluvio/.cargo/bin:${PATH}" +## Add `fluvio` to PATH +ENV PATH="/home/fluvio/.fluvio/bin:${PATH}" # Install dependencies RUN pip install --upgrade pip diff --git a/embeds/client-examples/python/hello-python-smartmodule.py b/embeds/client-examples/python/hello-python-smartmodule.py new file mode 100644 index 00000000..da74d813 --- /dev/null +++ b/embeds/client-examples/python/hello-python-smartmodule.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +import os +from datetime import datetime +from fluvio import Fluvio, Offset, ConsumerCoonfig + +TOPIC_NAME = "hello-python-smartmodule" +PARTITION = 0 + +# This is an example of a basic Fluvio workflow in Python +# +# 1. Create a topic to store data in via CLI +# 2. Establish a connection to the Fluvio cluster +# 3. Create a producer and send some bytes +# 4. Create a consumer, and stream the data back +if __name__ == "__main__": + # Currently the Python client does not support creating topics + # Using the fluvio CLI + os.popen("fluvio topic create {}".format(TOPIC_NAME)) + + # Connect to cluster + fluvio = Fluvio.connect() + + # Produce to topic + producer = fluvio.topic_producer(TOPIC_NAME) + producer.send_string("Hello World! - Time is: {}".format(datetime.now())) + + # Consume from topic + # We're just going to get the last record + consumer = fluvio.partition_consumer(TOPIC_NAME, PARTITION) + + + # Create a ConsumerConfig using your "uppercase-map" smartmodule + config = ConsumerConfig() + config.smartmodule(name="uppercase-map") + + for record in consumer.stream_with_config(Offset.from_end(0), config): + print("{}".format(record.value_string())) + break diff --git a/embeds/client-examples/python/python-consume.py b/embeds/client-examples/python/python-consume.py new file mode 100644 index 00000000..13e5ac20 --- /dev/null +++ b/embeds/client-examples/python/python-consume.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +from fluvio import Fluvio, Offset + +TOPIC_NAME = "python-data" +PARTITION = 0 + +if __name__ == "__main__": + # Connect to cluster + fluvio = Fluvio.connect() + + # Consume last 10 records from topic + consumer = fluvio.partition_consumer(TOPIC_NAME, PARTITION) + for idx, record in enumerate( consumer.stream(Offset.from_end(10)) ): + print("{}".format(record.value_string())) + + if idx >= 9: + break \ No newline at end of file diff --git a/embeds/client-examples/python/python-produce.py b/embeds/client-examples/python/python-produce.py new file mode 100644 index 00000000..d454b076 --- /dev/null +++ b/embeds/client-examples/python/python-produce.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +from datetime import datetime +from fluvio import Fluvio + +TOPIC_NAME = "python-data" +PARTITION = 0 + +if __name__ == "__main__": + # Connect to cluster + fluvio = Fluvio.connect() + + # Produce 10 records to topic + producer = fluvio.topic_producer(TOPIC_NAME) + for x in range(10): + producer.send_string("{}: timestamp: {}".format(x, datetime.now())) + + # Flush the last entry + producer.flush() \ No newline at end of file diff --git a/embeds/client-examples/rust/Cargo.lock b/embeds/client-examples/rust/Cargo.lock index ddb317e5..0e0b00eb 100644 --- a/embeds/client-examples/rust/Cargo.lock +++ b/embeds/client-examples/rust/Cargo.lock @@ -1513,13 +1513,14 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.20.1" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "pin-project-lite", "tokio-macros", + "windows-sys", ] [[package]] @@ -1779,6 +1780,63 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + [[package]] name = "ws_stream_wasm" version = "0.7.3" diff --git a/embeds/client-examples/rust/Dockerfile b/embeds/client-examples/rust/Dockerfile index 3ccf634e..f23bfd3f 100644 --- a/embeds/client-examples/rust/Dockerfile +++ b/embeds/client-examples/rust/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.65.0 +FROM rust:1.66.0 # Run as the `fluvio` user instead of root ENV USER=fluvio diff --git a/embeds/connectors-old/catfacts-basic-connector-with-transform.yaml b/embeds/connectors-old/catfacts-basic-connector-with-transform.yaml new file mode 100644 index 00000000..489462ec --- /dev/null +++ b/embeds/connectors-old/catfacts-basic-connector-with-transform.yaml @@ -0,0 +1,17 @@ +# catfacts-basic-connector-with-transform.yml +version: 0.4.3 +name: cat-facts-transformed +type: http-source +topic: cat-facts-data-transformed + +parameters: + endpoint: https://catfact.ninja/fact + interval: 30s + +transforms: + - uses: infinyon/jolt@0.1.0 + with: + spec: + - operation: default + spec: + source: "http" diff --git a/embeds/connectors-old/catfacts-basic-connector.yaml b/embeds/connectors-old/catfacts-basic-connector.yaml new file mode 100644 index 00000000..d61f1ca1 --- /dev/null +++ b/embeds/connectors-old/catfacts-basic-connector.yaml @@ -0,0 +1,9 @@ +# catfacts-basic-connector.yml +version: 0.4.3 +name: cat-facts +type: http-source +topic: cat-facts-data + +parameters: + endpoint: https://catfact.ninja/fact + interval: 30s diff --git a/embeds/connectors/developer-guide/python-connector/Dockerfile b/embeds/connectors-old/developer-guide/python-connector/Dockerfile similarity index 100% rename from embeds/connectors/developer-guide/python-connector/Dockerfile rename to embeds/connectors-old/developer-guide/python-connector/Dockerfile diff --git a/embeds/connectors/developer-guide/python-connector/get-cat-facts.py b/embeds/connectors-old/developer-guide/python-connector/get-cat-facts.py similarity index 100% rename from embeds/connectors/developer-guide/python-connector/get-cat-facts.py rename to embeds/connectors-old/developer-guide/python-connector/get-cat-facts.py diff --git a/embeds/connectors/inbound-examples/inbound-http.yaml b/embeds/connectors-old/inbound-examples/inbound-http.yaml similarity index 91% rename from embeds/connectors/inbound-examples/inbound-http.yaml rename to embeds/connectors-old/inbound-examples/inbound-http.yaml index ca06f24f..2ae84d67 100644 --- a/embeds/connectors/inbound-examples/inbound-http.yaml +++ b/embeds/connectors-old/inbound-examples/inbound-http.yaml @@ -1,4 +1,4 @@ -version: 0.4.2 +version: 0.4.3 name: cat-facts type: http-source topic: cat-facts diff --git a/embeds/connectors/inbound-examples/inbound-kafka.yaml b/embeds/connectors-old/inbound-examples/inbound-kafka.yaml similarity index 91% rename from embeds/connectors/inbound-examples/inbound-kafka.yaml rename to embeds/connectors-old/inbound-examples/inbound-kafka.yaml index 0ff5a7a8..4fce3be4 100644 --- a/embeds/connectors/inbound-examples/inbound-kafka.yaml +++ b/embeds/connectors-old/inbound-examples/inbound-kafka.yaml @@ -1,4 +1,4 @@ -version: 0.3.1 +version: 0.3.2 name: my-inbound-kafka type: kafka-source topic: fluvio-output-topic diff --git a/embeds/connectors/inbound-examples/inbound-mqtt.yaml b/embeds/connectors-old/inbound-examples/inbound-mqtt.yaml similarity index 90% rename from embeds/connectors/inbound-examples/inbound-mqtt.yaml rename to embeds/connectors-old/inbound-examples/inbound-mqtt.yaml index 5a4858c4..9b0b07f7 100644 --- a/embeds/connectors/inbound-examples/inbound-mqtt.yaml +++ b/embeds/connectors-old/inbound-examples/inbound-mqtt.yaml @@ -1,4 +1,4 @@ -version: 0.5.2 +version: 0.5.3 name: my-mqtt type: mqtt-source topic: mqtt-topic diff --git a/embeds/connectors/inbound-examples/inbound-postgres.yaml b/embeds/connectors-old/inbound-examples/inbound-postgres.yaml similarity index 100% rename from embeds/connectors/inbound-examples/inbound-postgres.yaml rename to embeds/connectors-old/inbound-examples/inbound-postgres.yaml diff --git a/embeds/connectors/outbound-examples/outbound-dynamodb.yaml b/embeds/connectors-old/outbound-examples/outbound-dynamodb.yaml similarity index 100% rename from embeds/connectors/outbound-examples/outbound-dynamodb.yaml rename to embeds/connectors-old/outbound-examples/outbound-dynamodb.yaml diff --git a/embeds/connectors/outbound-examples/outbound-kafka.yaml b/embeds/connectors-old/outbound-examples/outbound-kafka.yaml similarity index 99% rename from embeds/connectors/outbound-examples/outbound-kafka.yaml rename to embeds/connectors-old/outbound-examples/outbound-kafka.yaml index b8a0a587..2e44e674 100644 --- a/embeds/connectors/outbound-examples/outbound-kafka.yaml +++ b/embeds/connectors-old/outbound-examples/outbound-kafka.yaml @@ -1,6 +1,6 @@ name: my-outbound-kafka type: kafka-sink -version: 0.3.2 +version: 0.3.3 parameters: kafka-url: kafka-ssl.host.tld:9092 security-protocol: 'SSL' diff --git a/embeds/connectors/outbound-examples/outbound-postgres.yaml b/embeds/connectors-old/outbound-examples/outbound-postgres.yaml similarity index 100% rename from embeds/connectors/outbound-examples/outbound-postgres.yaml rename to embeds/connectors-old/outbound-examples/outbound-postgres.yaml diff --git a/embeds/connectors/outbound-examples/outbound-slack.yaml b/embeds/connectors-old/outbound-examples/outbound-slack.yaml similarity index 100% rename from embeds/connectors/outbound-examples/outbound-slack.yaml rename to embeds/connectors-old/outbound-examples/outbound-slack.yaml diff --git a/embeds/connectors/outbound-examples/outbound-sql.yaml b/embeds/connectors-old/outbound-examples/outbound-sql.yaml similarity index 97% rename from embeds/connectors/outbound-examples/outbound-sql.yaml rename to embeds/connectors-old/outbound-examples/outbound-sql.yaml index 0eda709b..989fa531 100644 --- a/embeds/connectors/outbound-examples/outbound-sql.yaml +++ b/embeds/connectors-old/outbound-examples/outbound-sql.yaml @@ -1,6 +1,6 @@ name: fluvio-sql-connector type: sql-sink -version: 0.1.1 +version: 0.1.2 topic: json-topic create-topic: true parameters: diff --git a/embeds/connectors/catfacts-basic-connector-with-transform.yaml b/embeds/connectors/catfacts-basic-connector-with-transform.yaml index 5bdba11b..ac2296d2 100644 --- a/embeds/connectors/catfacts-basic-connector-with-transform.yaml +++ b/embeds/connectors/catfacts-basic-connector-with-transform.yaml @@ -1,12 +1,13 @@ # catfacts-basic-connector-with-transform.yml -version: 0.4.2 -name: cat-facts-transformed -type: http-source -topic: cat-facts-data-transformed +meta: + version: 0.1.0 + name: cat-facts-transformed + type: http-source + topic: cat-facts-data-transformed -parameters: +http: endpoint: https://catfact.ninja/fact - interval: 30s + interval: 10s transforms: - uses: infinyon/jolt@0.1.0 diff --git a/embeds/connectors/catfacts-basic-connector.yaml b/embeds/connectors/catfacts-basic-connector.yaml index 1bfaae1d..bdd8a7c8 100644 --- a/embeds/connectors/catfacts-basic-connector.yaml +++ b/embeds/connectors/catfacts-basic-connector.yaml @@ -1,9 +1,9 @@ -# catfacts-basic-connector.yml -version: 0.4.2 -name: cat-facts -type: http-source -topic: cat-facts-data - -parameters: - endpoint: https://catfact.ninja/fact - interval: 30s +meta: + version: 0.1.0 + name: cat-facts + type: http-source + topic: cat-facts + create-topic: true +http: + endpoint: "https://catfact.ninja/fact" + interval: 10s diff --git a/embeds/connectors/inbound/http.md b/embeds/connectors/inbound/http.md new file mode 100644 index 00000000..b659f4db --- /dev/null +++ b/embeds/connectors/inbound/http.md @@ -0,0 +1,96 @@ +# Fluvio HTTP Connector +Official Infinyon HTTP connector + +## Source Connector +Sources HTTP Responses given input HTTP request configuration options and `interval` x. + +Supports HTTP/1.0, HTTP/1.1, HTTP/2.0 protocols. + + +See [docs](https://www.fluvio.io/connectors/inbound/http/) here. +Tutorial for [HTTP to SQL Pipeline](https://www.fluvio.io/docs/tutorials/data-pipeline/). + +### Configuration +| Option | default | type | description | +| :------------| :--------------------------| :----- | :----------------------------------------------------------------------------------------- | +| interval | 10s | String | Interval between each HTTP Request. This is in the form of "1s", "10ms", "1m", "1ns", etc. | +| method | GET | String | GET, POST, PUT, HEAD | +| endpoint | - | String | HTTP URL endpoint | +| headers | - | String | Request header(s) Key=Value pairs | +| body | - | String | Request body e.g. in POST | +| user-agent | "fluvio/http-source 0.1.0" | String | Request user-agent | +| output_type | text | String | `text` = UTF-8 String Output, `json` = UTF-8 JSON Serialized String | +| output_parts | body | String | `body` = body only, `full` = all status, header and body parts | + +#### Record Type Output +| Matrix | Output | +| :---------------------------------------------------------- | :-------------------------------------- | +| output_type = text (default), output_parts = body (default) | Only the body of the HTTP Response | +| output_type = text (default), output_parts = full | The full HTTP Response | +| output_type = json, output_parts = body (default) | Only the "body" in JSON struct | +| output_type = json, output_parts = full | HTTP "status", "body" and "header" JSON | + + +### Usage Example + +This is an example of simple connector config file: + +```yaml +# config-example.yaml +meta: + version: 0.1.0 + name: cat-facts + type: http-source + topic: cat-facts + create-topic: true +http: + endpoint: "https://catfact.ninja/fact" + interval: 10s +``` + +The produced record in Fluvio topic will be: +```json +{ + "fact": "The biggest wildcat today is the Siberian Tiger. It can be more than 12 feet (3.6 m) long (about the size of a small car) and weigh up to 700 pounds (317 kg).", + "length": 158 +} +``` + +### Transformations +Fluvio HTTP Source Connector supports [Transformations](https://www.fluvio.io/docs/concepts/transformations-chain/). Records can be modified before sending to Fluvio topic. + +The previous example can be extended to add extra transformations to outgoing records: +```yaml +# config-example.yaml +meta: + version: 0.1.0 + name: cat-facts + type: http-source + topic: cat-facts + create-topic: true +http: + endpoint: "https://catfact.ninja/fact" + interval: 10s +transforms: + - uses: infinyon/jolt@0.1.0 + with: + spec: + - operation: default + spec: + source: "http-connector" + - operation: remove + spec: + length: "" +``` +In this case, additional transformation will be performed before records are sent to Fluvio topic: field `length` will be removed and +field `source` with string value `http-connector` will be added. + +Now produced records will have a different shape, for example: +```json +{ + "fact": "A cat has more bones than a human; humans have 206, and the cat - 230.", + "source": "http-connector" +} +``` + +Read more about [JSON to JSON transformations](https://www.fluvio.io/smartmodules/certified/jolt/). diff --git a/embeds/connectors/inbound/kafka.md b/embeds/connectors/inbound/kafka.md new file mode 100644 index 00000000..22548730 --- /dev/null +++ b/embeds/connectors/inbound/kafka.md @@ -0,0 +1,38 @@ +## Source Connector +This is a connector for taking data from a Kafka topic and sending to a Fluvio topic. + +See [docs](https://www.fluvio.io/connectors/inbound/kafka/) here. + +### Configuration + +| Opt | default | type | description | +| :--- | :--- | :--- | :---- | +| url | - | String | The url for the kafka connector | +| topic | - | String | The kafka topic | +| partition | 0 | Integer | The kafka partition | +| group | fluvio-kafka-source | String | The kafka consumer group | + +Example: +```yaml +meta: + version: 0.1.0 + name: my-kafka-connector + type: kafka-source + topic: kafka-topic + create-topic: true +kafka: + url: "localhost:9092" + topic: fluvio-topic +``` + +### Usage +To try out Kafka Sink connector locally, you can use Fluvio CDK tool: +```bash +fluvio install cdk + +cdk deploy -p kafka-source start --config crates/kafka-source/config-example.yaml +``` + + +## Transformations +Fluvio Kafka Connectors support [Transformations](https://www.fluvio.io/docs/concepts/transformations-chain/). diff --git a/embeds/connectors/inbound/mqtt.md b/embeds/connectors/inbound/mqtt.md new file mode 100644 index 00000000..e3db85f2 --- /dev/null +++ b/embeds/connectors/inbound/mqtt.md @@ -0,0 +1,137 @@ +# Fluvio MQTT Connector +Official Infinyon MQTT connector + +## Source Connector +Reads record from MQTT topic and writes to Fluvio topic. + +Supports MQTT V3.1.1 and V5 protocols. + + +See [docs](https://www.fluvio.io/connectors/inbound/mqtt/) here. +Tutorial for [MQTT to SQL Pipeline](https://www.fluvio.io/docs/tutorials/mqtt-to-sql/). + +### Configuration +| Option | default | type | description | +|:--------------------|:---------|:--------- |:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| timeout | 60s | Duration | mqtt broker connect timeout in seconds and nanoseconds | +| url | - | SecretString | MQTT url which includes schema, domain, port and credentials such as username and password. | +| topic | - | String | mqtt topic to subscribe and source events from | +| client_id | UUID V4 | String | mqtt client ID | +| payload_output_type | binary | String | controls how the output of `payload` field is produced | + +`url` option with type `SecretString` can be set as raw string value: +```yaml +url: "mqtt://test.mosquitto.org/" +``` +or, as a reference to a secret with the given name: +```yaml +url: + secret: + name: "URL_SECRET_NAME" +``` + + +#### Record Type Output + +JSON Serialized string with fields `mqtt_topic` and `payload` + +#### Payload Output Type + +| Value | Output | +|:-------|:-----------------------------| +| binary | Array of bytes | +| json | UTF-8 JSON Serialized String | + +### Usage Example + +This is an example of connector config file: + +```yaml +# config-example.yaml +meta: + version: 0.1.0 + name: my-mqtt-connector + type: mqtt-source + topic: mqtt-topic + create-topic: true +mqtt: + url: "mqtt://test.mosquitto.org/" + topic: "mqtt-to-fluvio" + client_id: "my_mqtt" + timeout: + secs: 30 + nanos: 0 + payload_output_type: json +``` + +Run connector locally using `cdk` tool (from root directory or any sub-directory): +```bash +fluvio install cdk + +cdk deploy start --config config-example.yaml + +cdk deploy list # to see the status +cdk deploy log my-mqtt-connector # to see connector's logs +``` + +Install MQTT Client such as +```bash +# for mac , this takes while.... +brew install mosquitto +``` + +Insert records: +```bash +mosquitto_pub -h test.mosquitto.org -t mqtt-to-fluvio -m '{"device": {"device_id":1, "name":"device1"}}' +``` + +The produced record in Fluvio topic will be: +```json +{ + "mqtt_topic": "mqtt-to-fluvio", + "payload": { + "device": { + "device_id": 1, + "name": "device1" + } + } +} +``` +### Transformations +Fluvio MQTT Source Connector supports [Transformations](https://www.fluvio.io/docs/concepts/transformations-chain/). Records can be modified before sending to Fluvio topic. + +The previous example can be extended to add extra transformations to outgoing records: +```yaml +# config-example.yaml +meta: + version: 0.1.0 + name: my-mqtt-connector + type: mqtt-source + topic: mqtt-topic + create-topic: true +mqtt: + url: "mqtt://test.mosquitto.org/" + topic: "mqtt-to-fluvio" + client_id: "my_mqtt" + timeout: + secs: 30 + nanos: 0 + payload_output_type: json +transforms: + - uses: infinyon/jolt@0.1.0 + with: + spec: + - operation: shift + spec: + payload: + device: "device" + - operation: default + spec: + source: "mqtt-connector" +``` +The object `device` in the resulting record will be "unwrapped" and the addition field `source` with value `mqtt-connector` +will be added. + +Read more about [JSON to JSON transformations](https://www.fluvio.io/smartmodules/certified/jolt/). + + diff --git a/embeds/connectors/outbound/kafka.md b/embeds/connectors/outbound/kafka.md new file mode 100644 index 00000000..4b73c118 --- /dev/null +++ b/embeds/connectors/outbound/kafka.md @@ -0,0 +1,70 @@ + +## Sink Connector +This is a connector for taking data from a Fluvio topic and sending to a Kafka topic. + +See [docs](https://www.fluvio.io/connectors/outbound/kafka/) here. + +### Configuration + +| Opt | default | type | description | +| :--- | :--- | :--- | :---- | +| url | - | String | The url for the kafka connector | +| topic | - | String | The kafka topic | +| partition | 0 | Integer | The kafka partition | +| create-topic | false | Boolean | Create or not a topic before start | +| options | - | Mapping | The kafka client options | +| security | - | Mapping | Optional. The kafka security config | + +#### Security configuration +| Option | default | type | description | +| :--- | :--- | :--- | :---- | +| security_protocol | ssl | String | The kafka security protocol | +| ssl_key | - | Mapping | The SSL key file to use | +| ssl_cert | - | Mapping | The SSL cert file to use | +| ssl_ca | - | Mapping | The SSL ca file to use | + +Parameters `ssl_key`, `ssl_cert` and `ssl_ca` can be defined via `file` - path to the file, or `pem` - content as string value. + +Example without security: +```yaml +meta: + version: 0.1.0 + name: my-kafka-connector + type: kafka-sink + topic: kafka-topic + create-topic: true +kafka: + url: "localhost:9092" + topic: fluvio-topic + create-topic: true +``` + +Example with security enabled: +```yaml +meta: + version: 0.1.0 + name: my-kafka-connector + type: kafka-sink + topic: kafka-topic + create-topic: true +kafka: + url: "KAFKA_BROKER_URL" + topic: fluvio-topic + create-topic: true + security: + ssl_key: + file: /path/to/file + ssl_cert: + pem: "CONTENT IN PEM FORMAT" + ssl_ca: + file: /path/to/file + security_protocol: ssl +``` + +### Usage +To try out Kafka Sink connector locally, you can use Fluvio CDK tool: +```bash +fluvio install cdk + +cdk deploy -p kafka-sink start --config crates/kafka-sink/config-example.yaml +``` diff --git a/embeds/connectors/outbound/sql.md b/embeds/connectors/outbound/sql.md new file mode 100644 index 00000000..5e884f17 --- /dev/null +++ b/embeds/connectors/outbound/sql.md @@ -0,0 +1,115 @@ +# Fluvio SQL Sink connector +The SQL Sink connector reads records from Fluvio topic, applies configured transformations, and +sends new records to the SQL database (via `INSERT` statements). + +## Supported databases +1. PostgreSQL +2. SQLite + +### Data types +| Model | PostgreSQL | SQLite | +|:----------------|:-----------------------------|:-------------| +| Bool | BOOL | BOOLEAN | +| Char | CHAR | INTEGER | +| SmallInt | SMALLINT, SMALLSERIAL, INT2 | INTEGER | +| Int | INT, SERIAL, INT4 | INTEGER | +| BigInt | BIGINT, BIGSERIAL, INT8 | BIGINT, INT8 | +| Float | REAL, FLOAT4 | REAL | +| DoublePrecision | DOUBLE PRECISION, FLOAT8 | REAL | +| Text | VARCHAR, CHAR(N), TEXT, NAME | TEXT | +| Bytes | BYTEA | BLOB | +| Numeric | NUMERIC | REAL | +| Timestamp | TIMESTAMP | DATETIME | +| Date | DATE | DATE | +| Time | TIME | TIME | +| Uuid | UUID | BLOB, TEXT | +| Json | JSON, JSONB | TEXT | + +## Transformations +The SQL Sink connector expects the data in [Fluvio SQL Model](./crates/fluvio-model-sql/README.md) in JSON format. +In order to work with different data formats or data structures, `transformations` can be applied. +The transformation is a SmartModule pulled from the SmartModule Hub. Transformations are chained according to the order +in the config. If a SmartModule requires configuration, it is passed via `with` section of `transforms` entry. + +## Configuration +| Option | default | type | description | +|:-------------|:--------| :--- |:------------------------------------------------------| +| url | - | String | SQL database conection url | + +### Basic example: +```yaml +meta: + version: 0.1.0 + name: my-sql-connector + type: sql-sink + topic: sql-topic + create-topic: true +sql: + url: 'postgresql://USERNAME:PASSWORD@HOST:PORT/DB_NAME' +``` + +## Usage Example +Let's look at the example of the connector with one transformation named [infinyon/json-sql](https://github.com/infinyon/fluvio-connectors/blob/main/smartmodules/json-sql/README.md). The transformation takes +records in JSON format and creates SQL insert operation to `topic_message` table. The value from `device.device_id` +JSON field will be put to `device_id` column and the entire json body to `record` column. + +The JSON record: +```json +{ + "device": { + "device_id": 1 + } +} +``` + +The SQL database (Postgres): +``` +CREATE TABLE topic_message (device_id int, record json); +``` + +Connector configuration file: +```yaml +# connector-config.yaml +meta: + version: latest + name: json-sql-connector + type: sql-sink + topic: sql-topic + create-topic: true +sql: + url: 'postgresql://USERNAME:PASSWORD@HOST:PORT/DB_NAME' +transforms: + - uses: infinyon/json-sql + with: + mapping: + table: "topic_message" + map-columns: + "device_id": + json-key: "device.device_id" + value: + type: "int" + default: "0" + required: true + "record": + json-key: "$" + value: + type: "jsonb" + required: true +``` + +You can use Fluvio `cdk` tool to deploy the connector: +```bash +fluvio install cdk +``` +and then: +```bash +cdk deploy start --config connector-config.yaml +``` +To delete the connector run: +```bash +cdk deploy shutdown --config connector-config.yaml + +``` +After you run the connector you will see records in your database table. + +See more in our [Build MQTT to SQL Pipeline](https://www.fluvio.io/docs/tutorials/mqtt-to-sql/) and [Build HTTP to SQL Pipeline](https://www.fluvio.io/docs/tutorials/data-pipeline/) tutorials. diff --git a/embeds/deprecation-notice/connectors-old.txt b/embeds/deprecation-notice/connectors-old.txt new file mode 100644 index 00000000..a85f8958 --- /dev/null +++ b/embeds/deprecation-notice/connectors-old.txt @@ -0,0 +1,4 @@ +{{}} +These connector docs are deprecated. They may not represent the current functionality, and may go away soon.

                          +Please refer to the [current connector docs]({{}}) instead for up-to-date information. +{{                          }} \ No newline at end of file diff --git a/embeds/templates/connector-template.yaml b/embeds/templates/connector-template.yaml index aa9bbfc4..67741e36 100644 --- a/embeds/templates/connector-template.yaml +++ b/embeds/templates/connector-template.yaml @@ -1,61 +1,59 @@ # connector.yaml -# The name attached to this instance of connector -# This is the name seen when listing the connectors -# -name: - -# The version of the connector type to use -# See the individual connector docs for the current release version -# -version: - -# The Connector type -# -# Inbound Connector choices: -# * http-source -# * kafka-source -# * mqtt-source -# -# Outbound Connector choices: -# * kafka-sink -# * sql-sink -# -type: - -# The topic to produce to or consume from -# -topic: - -# Type specific parameters -# A list of parameter keys -# See the individual connector docs -# -parameters: -# example-param: - -# Type specific secrets -# A list of secret keys -# See individual connector docs -# -#secrets: -# EXAMPLE_SECRET: - -# Optional content for producer type connector -# -#producer: -# linger: -# batch-size: -# compression: - -# Optional content for consumer type connector -# -#consumer: -# partition: - +# Config that is common to all connectors. +# It contains common configurations like +# connector name, connector type and version +meta: + # The name attached to this instance of connector + # This is the name seen when listing the connectors + # + name: + + # The version of the connector type to use + # See the individual connector docs for the current release version + # + version: + + # The Connector type + # + # Inbound Connector choices: + # * http-source + # * kafka-source + # * mqtt-source + # + # Outbound Connector choices: + # * kafka-sink + # * sql-sink + # + type: + + # The topic to produce to or consume from + # + topic: + + # Optional content for producer type connector + # + #producer: + # linger: + # batch-size: + # compression: + + # Optional content for consumer type connector + # + #consumer: + # partition: # Optional list of transformations # #transforms: # - uses: smartmodule_name # with: # param_name: param_value + +# Type specific configs +# key depends on connector +# : +# foo: bar +# +# eg. +# http: +# endpoint: https://example.com diff --git a/embeds/tutorials/data-pipeline/sql-basic.yml b/embeds/tutorials/data-pipeline/sql-basic.yml index 0b59fd60..aaed2739 100644 --- a/embeds/tutorials/data-pipeline/sql-basic.yml +++ b/embeds/tutorials/data-pipeline/sql-basic.yml @@ -1,13 +1,14 @@ # sql.yaml -name: simple-cat-facts-sql -type: sql-sink -version: 0.1.1 -topic: cat-facts-data -parameters: - database-url: "postgres://user:password@db.postgreshost.example/dbname" - rust_log: "sql_sink=INFO,sqlx=WARN" +meta: + name: simple-cat-facts-sql + type: sql-sink + version: 0.1.0 + topic: cat-facts +sql: + url: "postgres://user:password@db.postgreshost.example/dbname" transforms: - uses: infinyon/json-sql@0.1.0 + invoke: insert with: mapping: table: "animalfacts" diff --git a/embeds/tutorials/data-pipeline/sql-transform.yml b/embeds/tutorials/data-pipeline/sql-transform.yml index c03a1078..b2aba493 100644 --- a/embeds/tutorials/data-pipeline/sql-transform.yml +++ b/embeds/tutorials/data-pipeline/sql-transform.yml @@ -1,11 +1,11 @@ # sql-chain.yaml -name: transform-cat-facts-sql -type: sql-sink -version: 0.1.1 -topic: cat-facts-data -parameters: - database-url: "postgres://user:password@db.postgreshost.example/dbname" - rust_log: "sql_sink=INFO,sqlx=WARN" +meta: + name: transform-cat-facts-sql + type: sql-sink + version: 0.1.0 + topic: cat-facts-data +sql: + url: "postgres://user:password@db.postgreshost.example/dbname" transforms: - uses: infinyon/jolt@0.1.0 with: @@ -20,6 +20,7 @@ transforms: type: "cat" - uses: infinyon/json-sql@0.1.0 with: + invoke: insert mapping: table: "animalfacts" map-columns: diff --git a/embeds/tutorials/mqtt-to-sql/mqtt-chain.yml b/embeds/tutorials/mqtt-to-sql/mqtt-chain.yml index 5d08e452..6ffa788f 100644 --- a/embeds/tutorials/mqtt-to-sql/mqtt-chain.yml +++ b/embeds/tutorials/mqtt-to-sql/mqtt-chain.yml @@ -1,15 +1,20 @@ # mqtt.yml -version: 0.5.2 -name: fluvio-mqtt-connector -type: mqtt-source -topic: mqtt-topic -direction: source -create-topic: true -parameters: - mqtt_topic: "ag-mqtt-topic" +meta: + version: 0.1.0 + name: fluvio-mqtt-connector + type: mqtt-source + topic: mqtt-topic + direction: source + create-topic: true +mqtt: + url: "mqtt://test.mosquitto.org/" + topic: "ag-mqtt-topic" + client_id: "my_mqtt" + timeout: + secs: 30 + nanos: 0 payload_output_type: json -secrets: - MQTT_URL: mqtt://test.mosquitto.org/ + transforms: - uses: infinyon/jolt@0.1.0 with: diff --git a/embeds/tutorials/mqtt-to-sql/mqtt.yml b/embeds/tutorials/mqtt-to-sql/mqtt.yml index 6e3288c0..ede0f653 100644 --- a/embeds/tutorials/mqtt-to-sql/mqtt.yml +++ b/embeds/tutorials/mqtt-to-sql/mqtt.yml @@ -1,12 +1,16 @@ # mqtt.yml -version: 0.5.2 -name: fluvio-mqtt-connector -type: mqtt-source -topic: mqtt-topic -direction: source -create-topic: true -parameters: - mqtt_topic: "ag-mqtt-topic" +meta: + version: 0.1.0 + name: fluvio-mqtt-connector + type: mqtt-source + topic: mqtt-topic + direction: source + create-topic: true +mqtt: + url: "mqtt://test.mosquitto.org/" + topic: "ag-mqtt-topic" + client_id: "my_mqtt" + timeout: + secs: 30 + nanos: 0 payload_output_type: json -secrets: - MQTT_URL: mqtt://test.mosquitto.org/ diff --git a/embeds/tutorials/mqtt-to-sql/sql-chain.yml b/embeds/tutorials/mqtt-to-sql/sql-chain.yml index 09b3c3f8..59f2df69 100644 --- a/embeds/tutorials/mqtt-to-sql/sql-chain.yml +++ b/embeds/tutorials/mqtt-to-sql/sql-chain.yml @@ -1,11 +1,12 @@ # sql-chain.yml -name: fluvio-sql-connector-chain -type: sql-sink -version: 0.1.1 -topic: mqtt-topic -create-topic: true -parameters: - database-url: "postgres://user:password@db.postgreshost.example/dbname" +meta: + name: fluvio-sql-connector-chain + type: sql-sink + version: 0.1.0 + topic: mqtt-topic + create-topic: true +sql: + url: "postgres://user:password@db.postgreshost.example/dbname" rust_log: "sql_sink=INFO,sqlx=WARN" transforms: - uses: infinyon/jolt@0.1.0 @@ -21,6 +22,7 @@ transforms: type: "mobile" - uses: infinyon/json-sql@0.1.0 with: + invoke: insert mapping: table: "topic_message" map-columns: diff --git a/embeds/tutorials/mqtt-to-sql/sql-pre-transformed.yml b/embeds/tutorials/mqtt-to-sql/sql-pre-transformed.yml index 2430907e..c515d8e2 100644 --- a/embeds/tutorials/mqtt-to-sql/sql-pre-transformed.yml +++ b/embeds/tutorials/mqtt-to-sql/sql-pre-transformed.yml @@ -1,15 +1,17 @@ # sql-chain.yml -name: fluvio-sql-connector-chain -type: sql-sink -version: 0.1.1 -topic: mqtt-topic -create-topic: true -parameters: - database-url: "postgres://user:password@db.postgreshost.example/dbname" - rust_log: "sql_sink=INFO,sqlx=WARN" +meta: + name: fluvio-sql-connector-chain + type: sql-sink + version: 0.1.0 + topic: mqtt-topic + create-topic: true +sql: + url: "postgres://user:password@db.postgreshost.example/dbname" + transforms: - uses: infinyon/json-sql@0.1.0 with: + invoke: insert mapping: table: "topic_message" map-columns: diff --git a/embeds/tutorials/mqtt-to-sql/sql.yml b/embeds/tutorials/mqtt-to-sql/sql.yml index feb5542a..bdd265e3 100644 --- a/embeds/tutorials/mqtt-to-sql/sql.yml +++ b/embeds/tutorials/mqtt-to-sql/sql.yml @@ -1,15 +1,16 @@ # sql.yml -name: fluvio-sql-connector -type: sql-sink -version: 0.1.1 -topic: mqtt-topic -create-topic: true -parameters: - database-url: "postgres://user:password@db.postgreshost.example/dbname" - rust_log: "sql_sink=INFO,sqlx=WARN" +meta: + name: fluvio-sql-connector + type: sql-sink + version: 0.1.0 + topic: mqtt-topic + create-topic: true +sql: + url: "postgres://user:password@db.postgreshost.example/dbname" transforms: - uses: infinyon/json-sql@0.1.0 with: + invoke: insert mapping: table: "topic_message" map-columns: diff --git a/functions/newsletter/package-lock.json b/functions/newsletter/package-lock.json index 55864eac..f0f2fcc4 100644 --- a/functions/newsletter/package-lock.json +++ b/functions/newsletter/package-lock.json @@ -5,9 +5,9 @@ "requires": true, "dependencies": { "@netlify/functions": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@netlify/functions/-/functions-1.3.0.tgz", - "integrity": "sha512-hN/Fgpz8XIOBfsBPLYUMxVKBlCopgeqGB0popayicnmkFLnvKByTTMYgF01wcF9DBtBQdV0H2h1kPFpMl34I8w==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@netlify/functions/-/functions-1.4.0.tgz", + "integrity": "sha512-gy7ULTIRroc2/jyFVGx1djCmmBMVisIwrvkqggq5B6iDcInRSy2Tpkm+V5C63hKJVkNRskKWtLQKm9ecCaQTjA==", "requires": { "is-promise": "^4.0.0" } diff --git a/functions/newsletter/package.json b/functions/newsletter/package.json index 5c250ae4..073b9087 100644 --- a/functions/newsletter/package.json +++ b/functions/newsletter/package.json @@ -14,7 +14,7 @@ "author": "Netlify", "license": "MIT", "dependencies": { - "@netlify/functions": "^1.0.0", + "@netlify/functions": "^1.4.0", "@types/node": "^18.11.7", "node-fetch": "^3.3.0", "typescript": "^4.9.3" diff --git a/hugo-start.sh b/hugo-start.sh index 6ac25570..c5fbd723 100755 --- a/hugo-start.sh +++ b/hugo-start.sh @@ -1,14 +1,23 @@ #!/bin/bash +WAIT=20 + +printf "\n**********\n* This script is deprecated!\n* Instead run:\n* cargo run -- hugo\n**********\n" + +if [[ -z "$CI" ]]; then + printf "\nStarting in $WAIT seconds..." + sleep $WAIT +fi + hugo server \ ---watch \ ---verbose \ ---buildDrafts \ ---cleanDestinationDir \ ---disableFastRender \ ---buildFuture \ ---ignoreCache \ ---baseURL http://localhost \ ---appendPort \ ---navigateToChanged \ ---renderToDisk \ + --watch \ + --verbose \ + --buildDrafts \ + --cleanDestinationDir \ + --disableFastRender \ + --buildFuture \ + --ignoreCache \ + --baseURL http://localhost \ + --appendPort \ + --navigateToChanged \ + --renderToDisk #--port 1313 diff --git a/layouts/connectors-old/list.html b/layouts/connectors-old/list.html new file mode 100644 index 00000000..8ddc6266 --- /dev/null +++ b/layouts/connectors-old/list.html @@ -0,0 +1,13 @@ +{{ define "menu" }} + {{- partial "top-menu.html" (dict "page" . "menu" .Site.Menus.infinyon) -}} + {{- partial "menu.html" (dict "page" . "menu" .Site.Menus.fluvio "bottomBorder" true) -}} +{{ end }} + +{{ define "content" }} + {{ partial "docs/doc-page" . }} +{{ end }} + +{{ define "footer" }} + {{ partial "footer/footer.html" . }} + +{{ end }} \ No newline at end of file diff --git a/layouts/connectors-old/single.html b/layouts/connectors-old/single.html new file mode 100644 index 00000000..8ddc6266 --- /dev/null +++ b/layouts/connectors-old/single.html @@ -0,0 +1,13 @@ +{{ define "menu" }} + {{- partial "top-menu.html" (dict "page" . "menu" .Site.Menus.infinyon) -}} + {{- partial "menu.html" (dict "page" . "menu" .Site.Menus.fluvio "bottomBorder" true) -}} +{{ end }} + +{{ define "content" }} + {{ partial "docs/doc-page" . }} +{{ end }} + +{{ define "footer" }} + {{ partial "footer/footer.html" . }} + +{{ end }} \ No newline at end of file diff --git a/scripts/fluvio-cms/Cargo.toml b/scripts/fluvio-cms/Cargo.toml new file mode 100644 index 00000000..310b4ed4 --- /dev/null +++ b/scripts/fluvio-cms/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "fluvio-cms" +version = "0.0.0" +edition = "2021" +description = "A content management system CLI for fluvio.io" +publish = false + +[dependencies] +anyhow = "1.0.69" +clap = { version = "4.1.8", features = ["derive"] } +convert_case = "0.6.0" +lazy_static = "1.4.0" +strum = { version = "0.24.1", features = ["derive"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } +xshell = "0.2.3" diff --git a/scripts/fluvio-cms/src/connectors/cli.rs b/scripts/fluvio-cms/src/connectors/cli.rs new file mode 100644 index 00000000..1f46d13d --- /dev/null +++ b/scripts/fluvio-cms/src/connectors/cli.rs @@ -0,0 +1,175 @@ +use std::fs::{self, File}; +use std::io::{BufReader, Read, Write}; +use std::path::PathBuf; + +use anyhow::Result; +use clap::Parser; +use convert_case::{Case, Casing}; +use tracing::info; +use xshell::{cmd, Shell}; + +use super::{DataDirection, DataService, DocsLocation, INBOUND, OUTBOUND}; + +/// Update the reference docs for a connector. By default, update all +#[derive(Clone, Parser, Debug)] +pub struct ConnectorsOpt { + //#[clap(value_enum)] + //doc: DocScheme, + /// Service or protocol used by connector to join data with Fluvio cluster + #[clap(long)] + service: Option, + /// Direction data flows through connector wrt Fluvio cluster. Aliases: source, sink + #[clap(long)] + direction: Option, + /// Path to README on the local filesystem, + #[clap(long = "file", requires_all = ["service", "direction"])] + localfile: Option, + /// Use the production instance of Hub + #[clap(long, action)] + prod: bool, + /// Display shell command output + #[clap(long, action)] + verbose: bool, +} + +impl ConnectorsOpt { + pub fn run(&self) -> Result<()> { + let direction = self.direction.unwrap_or(DataDirection::All); + let protocol = self.service.unwrap_or(DataService::All); + let mut connectors = Vec::new(); + + match (direction, protocol) { + // All the connectors + (DataDirection::All, DataService::All) => { + info!("Update all the connectors"); + connectors.extend(INBOUND.values()); + connectors.extend(OUTBOUND.values()); + } + // All the inbound + (DataDirection::Inbound, DataService::All) => { + info!("Update all the inbound connectors"); + connectors.extend(INBOUND.values()); + } + // All the outbound + (DataDirection::Outbound, DataService::All) => { + info!("Update all the outbound connectors"); + connectors.extend(OUTBOUND.values()); + } + // All of one type + (DataDirection::All, proto) => { + info!("Update all the {proto:?} connectors"); + if let Some(inbound) = INBOUND.get(&proto) { + connectors.push(inbound); + } + if let Some(outbound) = OUTBOUND.get(&proto) { + connectors.push(outbound); + } + } + // Just one + (DataDirection::Inbound, proto) => { + info!("Update inbound {proto:?} connector"); + if let Some(inbound) = INBOUND.get(&proto) { + connectors.push(inbound); + } + } + (DataDirection::Outbound, proto) => { + info!("Update outbound {proto:?} connector"); + if let Some(outbound) = OUTBOUND.get(&proto) { + connectors.push(outbound); + } + } + } + + for connector in connectors { + let sh = Shell::new()?; + let temp_dir = sh.create_temp_dir()?; + if let Some(f) = &self.localfile { + let f = File::open(f)?; + let mut reader = BufReader::new(f); + let mut buffer = Vec::new(); + + // Read file into vector. + reader.read_to_end(&mut buffer)?; + } else { + sh.change_dir(temp_dir.path()); + + let hub_group = connector.hub_group; + let hub_pkg_name = connector.hub_pkg_name; + let hub_pkg_version = connector.hub_pkg_version; + + let hub_remote = if self.prod { + Vec::new() + } else { + vec!["--remote", "https://hub-dev.infinyon.cloud"] + }; + + let mut cmd = + cmd!(sh, + "fluvio hub connector download {hub_group}/{hub_pkg_name}@{hub_pkg_version}" + ); + cmd.set_ignore_stdout(!self.verbose); + cmd.set_ignore_stderr(!self.verbose); + cmd.args(hub_remote).run()?; + + let mut cmd = cmd!( + sh, + "tar xvf {hub_group}-{hub_pkg_name}-{hub_pkg_version}.ipkg" + ); + cmd.set_ignore_stdout(!self.verbose); + cmd.set_ignore_stderr(!self.verbose); + cmd.run()?; + + let mut cmd = cmd!(sh, "tar xvf manifest.tar.gz"); + cmd.set_ignore_stdout(!self.verbose); + cmd.set_ignore_stderr(!self.verbose); + cmd.run()?; + }; + + let readme = temp_dir.path().join("README.md"); + + self.write_ref_to_disk(connector, readme)?; + } + Ok(()) + } + + fn write_ref_to_disk(&self, connector: &DocsLocation, readme: PathBuf) -> Result<()> { + let embed_connector_readme = format!( + "embeds/connectors/{}/{}.md", + connector.direction, + connector.service.to_string().to_case(Case::Lower) + ); + let connector_hugo_template = format!( + "content/connectors/{}/{}.md", + connector.direction, + connector.service.to_string().to_case(Case::Lower) + ); + + let sh = Shell::new()?; + + info!("Write the README file to {embed_connector_readme}"); + + sh.copy_file(readme, &embed_connector_readme)?; + + // Write the content file template + info!("Write the content template to {connector_hugo_template}"); + + let mut content_file = fs::OpenOptions::new() + .create(true) // To create a new file + .write(true) + .truncate(true) + .open(connector_hugo_template)?; + + write!( + &mut content_file, + r#"--- +menu: {connector_name_title} +--- + +{{{{% inline-embed file="{embed}" %}}}}"#, + connector_name_title = connector.service, + embed = embed_connector_readme + )?; + + Ok(()) + } +} diff --git a/scripts/fluvio-cms/src/connectors/mod.rs b/scripts/fluvio-cms/src/connectors/mod.rs new file mode 100644 index 00000000..e6b318b0 --- /dev/null +++ b/scripts/fluvio-cms/src/connectors/mod.rs @@ -0,0 +1,113 @@ +pub mod cli; +pub use cli::ConnectorsOpt; + +use std::collections::HashMap; + +use clap::ValueEnum; +use lazy_static::lazy_static; +use strum::Display; + +lazy_static! { + pub static ref INBOUND: HashMap = { + let mut m = HashMap::new(); + m.insert( + DataService::Http, + DocsLocation { + hub_group: "infinyon", + hub_pkg_name: "http-source", + hub_pkg_version: "0.1.1", + service: DataService::Http, + direction: DataDirection::Inbound, + }, + ); + + m.insert( + DataService::Mqtt, + DocsLocation { + hub_group: "infinyon", + hub_pkg_name: "mqtt-source", + hub_pkg_version: "0.1.2", + service: DataService::Mqtt, + direction: DataDirection::Inbound, + }, + ); + + m.insert( + DataService::Kafka, + DocsLocation { + hub_group: "infinyon", + hub_pkg_name: "kafka-source", + hub_pkg_version: "0.1.1", + service: DataService::Kafka, + direction: DataDirection::Inbound, + }, + ); + + m + }; + pub static ref OUTBOUND: HashMap = { + let mut m = HashMap::new(); + + m.insert( + DataService::Sql, + DocsLocation { + hub_group: "infinyon", + hub_pkg_name: "sql-sink", + hub_pkg_version: "0.1.1", + service: DataService::Sql, + direction: DataDirection::Outbound, + }, + ); + + m.insert( + DataService::Kafka, + DocsLocation { + hub_group: "infinyon", + hub_pkg_name: "kafka-sink", + hub_pkg_version: "0.1.1", + service: DataService::Kafka, + direction: DataDirection::Outbound, + }, + ); + + m + }; +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +pub struct DocsLocation { + hub_group: &'static str, + hub_pkg_name: &'static str, + hub_pkg_version: &'static str, + service: DataService, + direction: DataDirection, +} + +/// Selector for the data protocol of connectors +#[derive(Clone, Debug, Default, ValueEnum, PartialEq, Eq, Copy, Hash, Display)] +#[clap(rename_all = "lowercase")] +#[strum(serialize_all = "UPPERCASE")] +pub enum DataService { + #[default] + All, + #[strum(serialize = "Kafka")] + Kafka, + Http, + Sql, + Mqtt, + //Salesforce, + //Amplitude, +} + +/// Selector for the data direction of connectors +#[derive(Clone, Debug, Default, ValueEnum, PartialEq, Eq, Copy, Hash, Display)] +#[clap(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum DataDirection { + #[default] + All, + #[clap(alias = "source")] + Inbound, + #[clap(alias = "sink")] + Outbound, +} diff --git a/scripts/fluvio-cms/src/hugo/mod.rs b/scripts/fluvio-cms/src/hugo/mod.rs new file mode 100644 index 00000000..a61881c1 --- /dev/null +++ b/scripts/fluvio-cms/src/hugo/mod.rs @@ -0,0 +1,20 @@ +use anyhow::Result; +use clap::Parser; +use xshell::{cmd, Shell}; + +/// Wrapper around the `hugo` cli in your PATH. Starts development server +#[derive(Clone, Parser, Debug)] +pub struct HugoOpt {} + +impl HugoOpt { + pub fn run(&self) -> Result<()> { + let sh = Shell::new()?; + + cmd!( + sh, + "hugo server --watch --verbose --buildDrafts --cleanDestinationDir --disableFastRender --buildFuture --ignoreCache --baseURL http://localhost --appendPort --navigateToChanged --renderToDisk" + ) + .run()?; + Ok(()) + } +} diff --git a/scripts/fluvio-cms/src/main.rs b/scripts/fluvio-cms/src/main.rs new file mode 100644 index 00000000..b8f6c684 --- /dev/null +++ b/scripts/fluvio-cms/src/main.rs @@ -0,0 +1,71 @@ +pub mod connectors; +use connectors::ConnectorsOpt; + +pub mod hugo; +use hugo::HugoOpt; + +use anyhow::Result; +use clap::Parser; +use tracing_subscriber::filter::{EnvFilter, LevelFilter}; + +// Not needed yet +// +///// Refers to the quadrants of Divio documentation system +///// https://documentation.divio.com/ +//#[derive(Clone, Debug, ValueEnum, PartialEq)] +//pub enum DocScheme { +// #[clap(alias = "t")] +// Tutorials, +// #[clap(alias = "h")] +// HowTo, +// #[clap(alias = "e")] +// Explanation, +// #[clap(aliases= ["r", "ref","refs"])] +// Reference, +//} + +/// A specific section or general task to perform on the site +#[derive(Clone, Parser, Debug)] +enum Subject { + #[clap(visible_aliases = ["connector", "c"])] + Connectors(ConnectorsOpt), + //Cli, + //Api, + //Docs, + //#[clap(alias = "sm")] + //SmartModules, + //#[clap(alias = "twif")] + //ThisWeekInFluvio, + //CheckLinks, + Hugo(HugoOpt), +} + +impl Subject { + fn run(&self) -> Result<()> { + match self { + Subject::Connectors(opt) => opt.run(), + Subject::Hugo(opt) => opt.run(), + } + } +} + +/// Content management CLI for fluvio.io for Fluvio devs +/// +/// Use CLI to +/// * Do routine content updates and/or repetitive edits +/// * Run checks in CI for content that requires testing or validation over releases +#[derive(Clone, Parser, Debug)] +struct Args { + #[clap(subcommand)] + subject: Subject, +} + +fn main() -> Result<()> { + let _ = tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env().add_directive(LevelFilter::INFO.into())) + .try_init(); + + let args = Args::parse(); + + args.subject.run() +} diff --git a/scripts/update-cli-help.sh b/scripts/update-cli-help.sh index dc50de10..496b1053 100755 --- a/scripts/update-cli-help.sh +++ b/scripts/update-cli-help.sh @@ -2,6 +2,9 @@ THIS_SCRIPT_DIR=$(dirname $0) +FORMATTER_CMD="column -c" +COLUMNS=80 + CLI_COMMANDS_YML=$(realpath "${THIS_SCRIPT_DIR}/../data/cli-commands.yml") OUTPUT_DIR=$(realpath "${THIS_SCRIPT_DIR}/../embeds/cli/help") @@ -9,8 +12,16 @@ yq eval '.cli-commands[]' $CLI_COMMANDS_YML | while read -r cmd; do printf '%s\n' "$cmd" # just call it `fluvio` and save as markdown - FILENAME=$(printf "$cmd\n" | sed 's/fluvio-stable/fluvio/' | tr ' ' '-').md + FILENAME=$(printf "$cmd\n" | sed 's/fluvio-stable/fluvio/' | sed 's/fluvio-latest/fluvio/' | tr ' ' '-').md echo "\`\`\`" >$OUTPUT_DIR/$FILENAME - eval "$cmd -h" | sed 's/fluvio-stable/fluvio/' >>$OUTPUT_DIR/$FILENAME + + # Run the command help option + # Pass through formatter to constrain column width + # Replace `fluvio-stable` with just `fluvio` + # Make default paths match the linux defaults + eval "$cmd -h" | + $FORMATTER_CMD $COLUMNS | + sed -e 's/fluvio-stable/fluvio/' -e 's/fluvio-latest/fluvio/' -e 's|/usr/local/var/log/fluvio|/tmp|' \ + >>$OUTPUT_DIR/$FILENAME echo -n "\`\`\`" >>$OUTPUT_DIR/$FILENAME done diff --git a/content/connectors/images/smart-connectors-extra.svg b/static/images/connectors/smart-connectors-extra.svg similarity index 100% rename from content/connectors/images/smart-connectors-extra.svg rename to static/images/connectors/smart-connectors-extra.svg diff --git a/content/connectors/images/smart-connectors.svg b/static/images/connectors/smart-connectors.svg similarity index 100% rename from content/connectors/images/smart-connectors.svg rename to static/images/connectors/smart-connectors.svg