diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f87db3c3ab..125baafacf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,9 +41,12 @@ jobs: - name: Cache/Restore Rust dependencies. uses: actions/cache@v2 with: + # See: https://doc.rust-lang.org/nightly/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci + # TODO: Try https://github.com/Swatinem/rust-cache path: | - ~/.cargo/registry - ~/.cargo/git + ~/.cargo/registry/cache + ~/.cargo/registry/index + ~/.cargo/git/db target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} diff --git a/.gitmodules b/.gitmodules index 7a0b2b456f..2b312ca11b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "json/tests/official"] - path = json/tests/official + path = crates/json/tests/official url = https://github.com/json-schema-org/JSON-Schema-Test-Suite.git diff --git a/Cargo.lock b/Cargo.lock index f9f35c7498..be7370c731 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,10 +1,16 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "ahash" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6789e291be47ace86a60303502173d84af8327e3627ecf334356ee0f87a164c" + [[package]] name = "aho-corasick" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b476ce7103678b0c6d3d395dbbae31d48ff910bd28be979ba5d48c6351131d0d" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" dependencies = [ "memchr", ] @@ -20,15 +26,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1fd36ffbb1fb7c834eac128ea8d0e310c5aeb635548f9d58861e1308d46e71c" - -[[package]] -name = "arc-swap" -version = "0.4.7" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" +checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" [[package]] name = "arrayref" @@ -38,9 +38,9 @@ checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" [[package]] name = "arrayvec" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "async-stream" @@ -141,9 +141,9 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "blake2b_simd" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" dependencies = [ "arrayref", "arrayvec", @@ -224,11 +224,40 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "catalog" +version = "0.0.0" +dependencies = [ + "caseless", + "doc", + "humantime-serde", + "include_dir", + "insta", + "itertools 0.9.0", + "json", + "log", + "pretty_env_logger", + "protocol", + "regex", + "reqwest", + "rusqlite", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "strsim 0.10.0", + "tempfile", + "thiserror", + "unicode-normalization", + "url", + "yaml-merge-keys", +] + [[package]] name = "cc" -version = "1.0.61" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" +checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40" dependencies = [ "jobserver", ] @@ -305,9 +334,9 @@ dependencies = [ [[package]] name = "console" -version = "0.11.3" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c0994e656bba7b922d8dd1245db90672ffb701e684e45be58f20719d69abc5a" +checksum = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" dependencies = [ "encode_unicode", "lazy_static", @@ -319,9 +348,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" +checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" [[package]] name = "constant_time_eq" @@ -331,9 +360,9 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "core-foundation" -version = "0.7.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" dependencies = [ "core-foundation-sys", "libc", @@ -341,9 +370,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.7.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" [[package]] name = "cpuid-bool" @@ -515,6 +544,39 @@ dependencies = [ "once_cell", ] +[[package]] +name = "derive" +version = "0.0.0" +dependencies = [ + "catalog", + "doc", + "futures", + "http", + "hyper", + "hyper-tls", + "hyperlocal", + "insta", + "itertools 0.9.0", + "json", + "log", + "mime", + "pin-utils", + "prost", + "protocol", + "rocksdb", + "rusqlite", + "serde", + "serde_json", + "serve", + "tempfile", + "thiserror", + "tokio", + "tonic", + "tower", + "url", + "uuid", +] + [[package]] name = "derive_builder" version = "0.9.0" @@ -585,6 +647,25 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "doc" +version = "0.0.0" +dependencies = [ + "insta", + "itertools 0.9.0", + "json", + "lazy_static", + "quickcheck", + "quickcheck_macros", + "regex", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tinyvec", + "url", +] + [[package]] name = "dtoa" version = "0.4.6" @@ -611,11 +692,11 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -644,101 +725,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "estuary" -version = "0.1.0-unknown-development-version" -dependencies = [ - "ansi_term", - "anyhow", - "async-stream", - "async-trait", - "atty", - "bytes", - "caseless", - "chrono", - "clap", - "estuary_json", - "estuary_protocol", - "futures", - "fxhash", - "headers", - "http", - "http-body", - "humantime-serde", - "hyper", - "hyper-tls", - "hyperlocal", - "include_dir", - "insta", - "itertools 0.9.0", - "lazy_static", - "libc", - "log", - "mime", - "percent-encoding", - "pin-utils", - "pretty_env_logger", - "prost", - "prost-types", - "quickcheck", - "quickcheck_macros", - "regex", - "reqwest", - "rocksdb", - "rusqlite", - "schemars", - "serde", - "serde_json", - "serde_yaml", - "skim", - "strsim 0.10.0", - "structopt", - "tempfile", - "thiserror", - "tinyvec", - "tokio", - "tonic", - "tower", - "tuikit", - "unicode-normalization", - "unicode-width", - "url", - "uuid", - "warp", - "yaml-merge-keys", -] - -[[package]] -name = "estuary_json" -version = "0.1.0" -dependencies = [ - "fxhash", - "glob", - "itertools 0.9.0", - "percent-encoding", - "regex", - "serde", - "serde_json", - "serde_yaml", - "thiserror", - "tinyvec", - "url", -] - -[[package]] -name = "estuary_protocol" -version = "0.1.0" -dependencies = [ - "humantime-serde", - "insta", - "prost", - "prost-types", - "serde", - "serde_json", - "tonic", - "tonic-build", -] - [[package]] name = "fake-simd" version = "0.1.2" @@ -763,6 +749,52 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" +[[package]] +name = "flow-worker" +version = "0.0.0" +dependencies = [ + "anyhow", + "catalog", + "derive", + "doc", + "futures", + "hyper", + "libc", + "log", + "pretty_env_logger", + "protocol", + "rocksdb", + "serve", + "structopt", + "tokio", + "tower", +] + +[[package]] +name = "flowctl" +version = "0.0.0" +dependencies = [ + "anyhow", + "atty", + "catalog", + "doc", + "futures", + "itertools 0.9.0", + "lazy_static", + "log", + "materialization", + "pretty_env_logger", + "runtime", + "rusqlite", + "schemars", + "serde_json", + "structopt", + "tempfile", + "testing", + "tokio", + "url", +] + [[package]] name = "fnv" version = "1.0.7" @@ -784,6 +816,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "form_urlencoded" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +dependencies = [ + "matches", + "percent-encoding", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -808,9 +850,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8e3078b7b2a8a671cb7a3d17b4760e4181ea243227776ba83fd043b4ca034e" +checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" dependencies = [ "futures-channel", "futures-core", @@ -823,9 +865,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a4d35f7401e948629c9c3d6638fb9bf94e0b2121e96c3b428cc4e631f3eb74" +checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" dependencies = [ "futures-core", "futures-sink", @@ -833,15 +875,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d674eaa0056896d5ada519900dbf97ead2e46a7b6621e8160d79e2f2e1e2784b" +checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" [[package]] name = "futures-executor" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc709ca1da6f66143b8c9bec8e6260181869893714e9b5a490b169b0414144ab" +checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" dependencies = [ "futures-core", "futures-task", @@ -850,15 +892,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc94b64bb39543b4e432f1790b6bf18e3ee3b74653c5449f63310e9a74b123c" +checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" [[package]] name = "futures-macro" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f57ed14da4603b2554682e9f2ff3c65d7567b53188db96cb71538217fc64581b" +checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -868,24 +910,24 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d8764258ed64ebc5d9ed185cf86a95db5cac810269c5d20ececb32e0088abbd" +checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" [[package]] name = "futures-task" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd26820a9f3637f1302da8bceba3ff33adbe53464b54ca24d4e2d4f1db30f94" +checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" dependencies = [ "once_cell", ] [[package]] name = "futures-util" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a894a0acddba51a2d49a6f4263b1e64b8c579ece8af50fa86503d52cd1eea34" +checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" dependencies = [ "futures-channel", "futures-core", @@ -894,7 +936,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project", + "pin-project 1.0.1", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -980,6 +1022,18 @@ name = "hashbrown" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" +dependencies = [ + "hashbrown", +] [[package]] name = "headers" @@ -1090,9 +1144,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.8" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" +checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" dependencies = [ "bytes", "futures-channel", @@ -1104,7 +1158,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project", + "pin-project 1.0.1", "socket2", "tokio", "tower-service", @@ -1150,7 +1204,7 @@ dependencies = [ "futures-util", "hex", "hyper", - "pin-project", + "pin-project 0.4.27", "tokio", ] @@ -1215,9 +1269,9 @@ dependencies = [ [[package]] name = "insta" -version = "0.16.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "617e921abc813f96a3b00958c079e7bf1e2db998f8a04f1546dd967373a418ee" +checksum = "02da3dc37ef840c304daca59a47197bbd7da007116b2244754d5bbf07dc0abb7" dependencies = [ "console", "difference", @@ -1284,6 +1338,23 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json" +version = "0.0.0" +dependencies = [ + "fxhash", + "glob", + "itertools 0.9.0", + "percent-encoding", + "regex", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tinyvec", + "url", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -1294,6 +1365,10 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "labels" +version = "0.0.0" + [[package]] name = "lazy_static" version = "1.4.0" @@ -1308,9 +1383,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" +checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" [[package]] name = "libloading" @@ -1336,9 +1411,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.17.3" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d90181c2904c287e5390186be820e5ef311a3c62edebb7d6ca3d6a48ce041d" +checksum = "64d31059f22935e6c31830db5249ba2b7ecd54fd73a9909286f0a67aa55c2fbd" dependencies = [ "cc", "pkg-config", @@ -1360,21 +1435,34 @@ dependencies = [ "cfg-if 0.1.10", ] -[[package]] -name = "lru-cache" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "matches" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "materialization" +version = "0.0.0" +dependencies = [ + "catalog", + "doc", + "insta", + "itertools 0.9.0", + "json", + "labels", + "log", + "prost-types", + "protocol", + "rusqlite", + "serde", + "serde_json", + "skim", + "thiserror", + "tuikit", + "unicode-width", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -1383,9 +1471,9 @@ checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "memchr" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" [[package]] name = "memoffset" @@ -1439,7 +1527,7 @@ checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" dependencies = [ "log", "mio", - "miow 0.3.5", + "miow 0.3.6", "winapi 0.3.9", ] @@ -1468,9 +1556,9 @@ dependencies = [ [[package]] name = "miow" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" dependencies = [ "socket2", "winapi 0.3.9", @@ -1502,9 +1590,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" +checksum = "6fcc7939b5edc4e4f86b1b4a04bb1498afaaf871b1a6691838ed06fcb48d3a3f" dependencies = [ "lazy_static", "libc", @@ -1554,9 +1642,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ "autocfg 1.0.1", "num-traits", @@ -1564,9 +1652,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ "autocfg 1.0.1", ] @@ -1583,9 +1671,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" +checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" [[package]] name = "opaque-debug" @@ -1660,7 +1748,16 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" dependencies = [ - "pin-project-internal", + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841" +dependencies = [ + "pin-project-internal 1.0.1", ] [[package]] @@ -1674,6 +1771,17 @@ dependencies = [ "syn", ] +[[package]] +name = "pin-project-internal" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.1.11" @@ -1694,9 +1802,9 @@ checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" [[package]] name = "ppv-lite86" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" [[package]] name = "pretty_env_logger" @@ -1734,9 +1842,9 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.18" +version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro-nested" @@ -1804,6 +1912,20 @@ dependencies = [ "prost", ] +[[package]] +name = "protocol" +version = "0.0.0" +dependencies = [ + "humantime-serde", + "insta", + "prost", + "prost-types", + "serde", + "serde_json", + "tonic", + "tonic-build", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -2052,9 +2174,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" dependencies = [ "aho-corasick", "memchr", @@ -2064,9 +2186,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.20" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" [[package]] name = "remove_dir_all" @@ -2138,20 +2260,39 @@ dependencies = [ "librocksdb-sys", ] +[[package]] +name = "runtime" +version = "0.0.0" +dependencies = [ + "chrono", + "futures", + "labels", + "log", + "prost-types", + "protocol", + "rusqlite", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tokio", + "tonic", +] + [[package]] name = "rusqlite" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57edf4c4cea4d7e0fab069acb5da9e8e8e5403c78abc81b1f37d83af02148ea5" +checksum = "7e3d4791ab5517217f51216a84a688b53c1ebf7988736469c538d02f46ddba68" dependencies = [ "bitflags", "fallible-iterator", "fallible-streaming-iterator", + "hashlink", "libsqlite3-sys", - "lru-cache", "memchr", "serde_json", - "time", + "smallvec", "url", ] @@ -2256,9 +2397,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "0.4.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" +checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69" dependencies = [ "bitflags", "core-foundation", @@ -2269,9 +2410,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "0.4.3" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" +checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b" dependencies = [ "core-foundation-sys", "libc", @@ -2333,9 +2474,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.13" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae3e2dd40a7cdc18ca80db804b7f461a39bb721160a85c9a1fa30134bf3c02a5" +checksum = "f7baae0a99f1a324984bcdc5f0718384c1f69775f1c7eec8b859b71b443e3fd7" dependencies = [ "dtoa", "linked-hash-map", @@ -2343,6 +2484,21 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "serve" +version = "0.0.0" +dependencies = [ + "http", + "http-body", + "hyper", + "hyperlocal", + "log", + "tempfile", + "tokio", + "tower", + "warp", +] + [[package]] name = "sha-1" version = "0.8.2" @@ -2357,12 +2513,12 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "170a36ea86c864a3f16dd2687712dd6646f7019f301e57537c7f4dc9f5916770" +checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" dependencies = [ "block-buffer 0.9.0", - "cfg-if 0.1.10", + "cfg-if 1.0.0", "cpuid-bool", "digest 0.9.0", "opaque-debug 0.3.0", @@ -2376,19 +2532,18 @@ checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" [[package]] name = "signal-hook-registry" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e12110bc539e657a646068aaf5eb5b63af9d0c1f7b29c97113fad80e15f035" +checksum = "ce32ea0c6c56d5eacaeb814fbed9960547021d3edd010ded1425f180536b20ab" dependencies = [ - "arc-swap", "libc", ] [[package]] name = "skim" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "141d16fb63a98e3ed0e4012546074967bffbdb94dc251f00f497b5343ad4beb5" +checksum = "1ac8dcb2f57eba70ce16c6844af6caa63dc642d404630aae5dc3b4cb48353962" dependencies = [ "beef", "bitflags", @@ -2418,11 +2573,17 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +[[package]] +name = "smallvec" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" + [[package]] name = "socket2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" +checksum = "7fd8b795c389288baa5f355489c65e71fd48a02104600d15c4cfbc561e9e429d" dependencies = [ "cfg-if 0.1.10", "libc", @@ -2480,9 +2641,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.46" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad5de3220ea04da322618ded2c42233d02baca219d6f160a3e9c87cda16c942" +checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" dependencies = [ "proc-macro2", "quote", @@ -2524,9 +2685,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.1.13" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" +checksum = "4bd2d183bd3fac5f5fe38ddbeb4dc9aec4a39a9d7d59e7491d900302da01cbe1" dependencies = [ "libc", "winapi 0.3.9", @@ -2541,6 +2702,29 @@ dependencies = [ "libc", ] +[[package]] +name = "testing" +version = "0.0.0" +dependencies = [ + "catalog", + "derive", + "doc", + "futures", + "itertools 0.9.0", + "json", + "log", + "percent-encoding", + "prost", + "protocol", + "runtime", + "rusqlite", + "serde", + "serde_json", + "thiserror", + "tonic", + "url", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -2552,18 +2736,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42" +checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" +checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" dependencies = [ "proc-macro2", "quote", @@ -2601,15 +2785,24 @@ dependencies = [ [[package]] name = "tinyvec" -version = "0.3.4" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" +checksum = "b78a366903f506d2ad52ca8dc552102ffdd3e937ba8a227f024dc1d1eae28575" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff" dependencies = [ "bytes", "fnv", @@ -2631,9 +2824,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" dependencies = [ "proc-macro2", "quote", @@ -2670,7 +2863,7 @@ checksum = "6d9e878ad426ca286e4dcae09cbd4e1973a7f8987d97570e2469703dd7f5720c" dependencies = [ "futures-util", "log", - "pin-project", + "pin-project 0.4.27", "tokio", "tungstenite", ] @@ -2705,7 +2898,7 @@ dependencies = [ "http-body", "hyper", "percent-encoding", - "pin-project", + "pin-project 0.4.27", "prost", "prost-derive", "tokio", @@ -2758,7 +2951,7 @@ dependencies = [ "futures-core", "futures-util", "indexmap", - "pin-project", + "pin-project 0.4.27", "rand 0.7.3", "slab", "tokio", @@ -2778,7 +2971,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4887dc2a65d464c8b9b66e0e4d51c2fd6cf5b3373afc72805b0a60bce00446a" dependencies = [ "futures-core", - "pin-project", + "pin-project 0.4.27", "tokio", "tower-layer", "tower-service", @@ -2792,7 +2985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f6b5000c3c54d269cc695dff28136bb33d08cbf1df2c48129e143ab65bf3c2a" dependencies = [ "futures-core", - "pin-project", + "pin-project 0.4.27", "tower-service", ] @@ -2809,7 +3002,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92c3040c5dbed68abffaa0d4517ac1a454cd741044f33ab0eefab6b8d1361404" dependencies = [ "futures-core", - "pin-project", + "pin-project 0.4.27", "tokio", "tower-layer", "tower-load", @@ -2824,7 +3017,7 @@ checksum = "8cc79fc3afd07492b7966d7efa7c6c50f8ed58d768a6075dd7ae6591c5d2017b" dependencies = [ "futures-core", "log", - "pin-project", + "pin-project 0.4.27", "tokio", "tower-discover", "tower-service", @@ -2837,7 +3030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f021e23900173dc315feb4b6922510dae3e79c689b74c089112066c11f0ae4e" dependencies = [ "futures-core", - "pin-project", + "pin-project 0.4.27", "tower-layer", "tower-service", ] @@ -2873,7 +3066,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6727956aaa2f8957d4d9232b308fe8e4e65d99db30f42b225646e86c9b6a952" dependencies = [ "futures-core", - "pin-project", + "pin-project 0.4.27", "tokio", "tower-layer", "tower-service", @@ -2891,7 +3084,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "127b8924b357be938823eaaec0608c482d40add25609481027b96198b2e4b31e" dependencies = [ - "pin-project", + "pin-project 0.4.27", "tokio", "tower-layer", "tower-service", @@ -2905,7 +3098,7 @@ checksum = "d1093c19826d33807c72511e68f73b4a0469a3f22c2bd5f7d5212178b4b89674" dependencies = [ "futures-core", "futures-util", - "pin-project", + "pin-project 0.4.27", "tower-service", ] @@ -2948,7 +3141,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" dependencies = [ - "pin-project", + "pin-project 0.4.27", "tracing", ] @@ -2986,7 +3179,7 @@ dependencies = [ "input_buffer", "log", "rand 0.7.3", - "sha-1 0.9.1", + "sha-1 0.9.2", "url", "utf-8", ] @@ -3026,18 +3219,18 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +checksum = "b7f98e67a4d84f730d343392f9bfff7d21e3fca562b9cb7a43b768350beeddc6" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" +checksum = "db8716a166f290ff49dabc18b44aa407cb7c6dbe1aa0971b44b8a24b0ca35aae" [[package]] name = "unicode-width" @@ -3059,10 +3252,11 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" dependencies = [ + "form_urlencoded", "idna", "matches", "percent-encoding", @@ -3150,7 +3344,7 @@ dependencies = [ "mime", "mime_guess", "multipart", - "pin-project", + "pin-project 0.4.27", "scoped-tls", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index e3d8b49612..a258a0c109 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,85 +1,14 @@ -[package] -name = "estuary" -# The Makefile determines the real version. This string is only used as a fallback when invoking -# cargo directly. -version = "0.1.0-unknown-development-version" -authors = ["Estuary Technologies, Inc. \u00A92020"] -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [workspace] -members = [ - "json", - "protocol-rs", -] - -[[bin]] -name = "flow-worker" -path = "cmd/flow-worker/main.rs" +members = [ "crates/*"] -[[bin]] -name = "flowctl" -path = "cmd/flowctl/main.rs" +[profile.dev] +# Disabling debug info speeds up builds a bunch, +# and we don't rely on it for debugging that much. +debug = 0 -[dependencies] -ansi_term = "^0.11" -anyhow = "^1.0" -async-stream = "^0.2" -async-trait = "0.1" -atty = "0.2" -bytes = "^0.5" -caseless = "^0.2" -chrono = { version = "0.4", features = ["serde"] } -clap = "^2.33" # Used only for CLI binaries. -estuary_json = { path = "json" } -estuary_protocol = { path = "protocol-rs" } -futures = "^0.3" -fxhash = "^0.2" -headers = "^0.3" -http = "^0.2" -http-body = "^0.3" -humantime-serde = "1.0" -hyper = "^0.13" -hyper-tls = "^0.4" -hyperlocal = "^0.7" -include_dir = { version = "0.6", default-features = false } -itertools = "^0.9" -lazy_static = "*" -libc = "^0.2" -log = "^0.4" -mime = "0.3" -percent-encoding = "2.1.0" -pin-utils = "^0.1" -pretty_env_logger = "^0.4" -prost = "0.6" -prost-types = "0.6" -regex = "^1.3" -reqwest = { version = "^0.10", default-features = false, features = ["blocking", "rustls-tls"] } -rocksdb = "^0.15" -rusqlite = { version = "^0.22", features = ["functions", "bundled", "url", "serde_json", "collation"] } -schemars = "^0.8" -serde = { version = "^1.0", features = ["derive"] } -serde_json = { version = "^1.0", features = ["raw_value"] } -serde_yaml = "^0.8" -strsim = "0.10.0" -structopt = "0.3" -skim = "0.9" -tempfile = "^3.1" -thiserror = "^1.0" -tinyvec = {version = "^0.3", features = ["alloc"]} -tokio = { version = "0.2", features = ["full"] } -tonic = { version = "0.3" } -tower = "0.3" -tuikit = "0.4" -unicode-normalization = "^0.1" -unicode-width = "0.1" -url = "^2.1" -uuid = "^0.8" -warp = "0.2" -yaml-merge-keys = { version = "^0.4", features = ["serde_yaml"] } +[profile.release] +incremental = true +debug = 0 # Set this to 1 or 2 to get more useful backtraces in debugger. -[dev-dependencies] -insta = "^0.16" -quickcheck = "^0.9" -quickcheck_macros = "^0.9" +[patch.'crates-io'] +# rowan = { path = "../rowan" } diff --git a/Makefile b/Makefile index d040480a56..53d1070bb8 100644 --- a/Makefile +++ b/Makefile @@ -140,8 +140,8 @@ go-install/%: go install -v --tags "${GO_BUILD_TAGS}" \ -ldflags "-X $${MBP}.Version=${VERSION} -X $${MBP}.BuildDate=${DATE}" $* -${GOBIN}/flow-ingester: go-install/github.com/estuary/flow/cmd/flow-ingester $(GO_PROTO_TARGETS) -${GOBIN}/flow-consumer: go-install/github.com/estuary/flow/cmd/flow-consumer $(GO_PROTO_TARGETS) +${GOBIN}/flow-ingester: go-install/github.com/estuary/flow/go/flow-ingester $(GO_PROTO_TARGETS) +${GOBIN}/flow-consumer: go-install/github.com/estuary/flow/go/flow-consumer $(GO_PROTO_TARGETS) ${GOBIN}/gazette: go-install/go.gazette.dev/core/cmd/gazette ${GOBIN}/gazctl: go-install/go.gazette.dev/core/cmd/gazctl @@ -197,11 +197,11 @@ install-tools: ${TOOLBIN}/protoc-gen-gogo ${TOOLBIN}/etcd ${TOOLBIN}/sqlite3 .PHONY: sql-test sql-test: ${TOOLBIN}/sqlite3 - ${ROOTDIR}/src/catalog/test_catalog.sh + ${ROOTDIR}/crates/catalog/src/test_catalog.sh .PHONY: rust-test rust-test: ${TOOLBIN}/sqlite3 - FLOW_VERSION=${VERSION} cargo test --all --locked + FLOW_VERSION=${VERSION} cargo test --locked .PHONY: build-test-catalog build-test-catalog: ${ROOTDIR}/catalog.db @@ -223,7 +223,7 @@ catalog-test: ${RUSTBIN} ${GOBIN}/flow-ingester ${GOBIN}/flow-consumer ${GOBIN}/ package: $(PACKAGE_TARGETS) .PHONY: docker-image -docker-image: +docker-image: package docker build \ --file ${ROOTDIR}/.devcontainer/release.Dockerfile \ --tag docker.pkg.github.com/estuary/flow/bin:${VERSION} \ diff --git a/crates/catalog/Cargo.toml b/crates/catalog/Cargo.toml new file mode 100644 index 0000000000..5f52c9b641 --- /dev/null +++ b/crates/catalog/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "catalog" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +json = { path = "../json", version = "0.0.0" } +doc = { path = "../doc", version = "0.0.0" } + +caseless = "*" +humantime-serde = "*" +include_dir = { version = "*", default-features = false } +itertools = "*" +log = "*" +regex = "*" +reqwest = { version = "*", default-features = false, features = ["blocking", "rustls-tls"] } +rusqlite = { version = "*", features = ["bundled", "collation", "column_decltype", "functions", "serde_json", "url"] } +schemars = "*" +serde = { version = "*", features = ["derive"] } +serde_json = { version = "*", features = ["raw_value"] } +serde_yaml = "*" +strsim = "*" +tempfile = "*" +thiserror = "*" +unicode-normalization = "*" +url = "*" +yaml-merge-keys = { version = "*", features = ["serde_yaml"] } + +[dev-dependencies] +protocol = { path = "../protocol", version = "0.0.0" } + +insta = "*" +pretty_env_logger = "*" \ No newline at end of file diff --git a/catalog-js-transformer-template/.eslintignore b/crates/catalog/catalog-js-transformer-template/.eslintignore similarity index 100% rename from catalog-js-transformer-template/.eslintignore rename to crates/catalog/catalog-js-transformer-template/.eslintignore diff --git a/catalog-js-transformer-template/.eslintrc.js b/crates/catalog/catalog-js-transformer-template/.eslintrc.js similarity index 100% rename from catalog-js-transformer-template/.eslintrc.js rename to crates/catalog/catalog-js-transformer-template/.eslintrc.js diff --git a/catalog-js-transformer-template/.prettierrc.js b/crates/catalog/catalog-js-transformer-template/.prettierrc.js similarity index 100% rename from catalog-js-transformer-template/.prettierrc.js rename to crates/catalog/catalog-js-transformer-template/.prettierrc.js diff --git a/catalog-js-transformer-template/package.json b/crates/catalog/catalog-js-transformer-template/package.json similarity index 100% rename from catalog-js-transformer-template/package.json rename to crates/catalog/catalog-js-transformer-template/package.json diff --git a/catalog-js-transformer-template/src/app.ts b/crates/catalog/catalog-js-transformer-template/src/app.ts similarity index 100% rename from catalog-js-transformer-template/src/app.ts rename to crates/catalog/catalog-js-transformer-template/src/app.ts diff --git a/catalog-js-transformer-template/src/catalog/collections.ts b/crates/catalog/catalog-js-transformer-template/src/catalog/collections.ts similarity index 100% rename from catalog-js-transformer-template/src/catalog/collections.ts rename to crates/catalog/catalog-js-transformer-template/src/catalog/collections.ts diff --git a/catalog-js-transformer-template/src/catalog/lambdas.ts b/crates/catalog/catalog-js-transformer-template/src/catalog/lambdas.ts similarity index 100% rename from catalog-js-transformer-template/src/catalog/lambdas.ts rename to crates/catalog/catalog-js-transformer-template/src/catalog/lambdas.ts diff --git a/catalog-js-transformer-template/src/catalog/registers.ts b/crates/catalog/catalog-js-transformer-template/src/catalog/registers.ts similarity index 100% rename from catalog-js-transformer-template/src/catalog/registers.ts rename to crates/catalog/catalog-js-transformer-template/src/catalog/registers.ts diff --git a/catalog-js-transformer-template/src/runtime/serve.ts b/crates/catalog/catalog-js-transformer-template/src/runtime/serve.ts similarity index 100% rename from catalog-js-transformer-template/src/runtime/serve.ts rename to crates/catalog/catalog-js-transformer-template/src/runtime/serve.ts diff --git a/catalog-js-transformer-template/src/runtime/types.ts b/crates/catalog/catalog-js-transformer-template/src/runtime/types.ts similarity index 100% rename from catalog-js-transformer-template/src/runtime/types.ts rename to crates/catalog/catalog-js-transformer-template/src/runtime/types.ts diff --git a/catalog-js-transformer-template/tsconfig.json b/crates/catalog/catalog-js-transformer-template/tsconfig.json similarity index 100% rename from catalog-js-transformer-template/tsconfig.json rename to crates/catalog/catalog-js-transformer-template/tsconfig.json diff --git a/src/catalog/catalog.sql b/crates/catalog/src/catalog.sql similarity index 100% rename from src/catalog/catalog.sql rename to crates/catalog/src/catalog.sql diff --git a/src/catalog/catalog_test.sql b/crates/catalog/src/catalog_test.sql similarity index 100% rename from src/catalog/catalog_test.sql rename to crates/catalog/src/catalog_test.sql diff --git a/src/catalog/catalog_test_golden.out b/crates/catalog/src/catalog_test_golden.out similarity index 100% rename from src/catalog/catalog_test_golden.out rename to crates/catalog/src/catalog_test_golden.out diff --git a/src/catalog/collection.rs b/crates/catalog/src/collection.rs similarity index 100% rename from src/catalog/collection.rs rename to crates/catalog/src/collection.rs diff --git a/src/catalog/content_type.rs b/crates/catalog/src/content_type.rs similarity index 100% rename from src/catalog/content_type.rs rename to crates/catalog/src/content_type.rs diff --git a/src/catalog/db.rs b/crates/catalog/src/db.rs similarity index 100% rename from src/catalog/db.rs rename to crates/catalog/src/db.rs diff --git a/src/catalog/derivation.rs b/crates/catalog/src/derivation.rs similarity index 99% rename from src/catalog/derivation.rs rename to crates/catalog/src/derivation.rs index 17f01b8f31..a267005caf 100644 --- a/src/catalog/derivation.rs +++ b/crates/catalog/src/derivation.rs @@ -1,6 +1,6 @@ use super::{specs, sql_params, Collection, Error, Lambda, Resource, Schema, Scope, Selector}; -use crate::catalog::inference; -use crate::doc::{validate, FullContext, SchemaIndex, Validator}; +use crate::inference; +use doc::{validate, FullContext, SchemaIndex, Validator}; use serde_json::Value; /// Derivation is a catalog Collection which is derived from other Collections. diff --git a/src/catalog/error.rs b/crates/catalog/src/error.rs similarity index 96% rename from src/catalog/error.rs rename to crates/catalog/src/error.rs index 160d878d84..5203bbf5da 100644 --- a/src/catalog/error.rs +++ b/crates/catalog/src/error.rs @@ -1,8 +1,8 @@ use super::ContentType; -use crate::catalog::extraction::KeyError; -use crate::doc::{self, inference}; -use estuary_json::schema; +use crate::extraction::KeyError; +use doc::{self, inference}; use itertools::Itertools; +use json::schema; use std::fmt; #[derive(thiserror::Error, Debug)] @@ -81,7 +81,7 @@ pub enum Error { FailedValidation(doc::FailedValidation), #[error(transparent)] - InvalidProjection(#[from] crate::catalog::projections::NoSuchLocationError), + InvalidProjection(#[from] crate::projections::NoSuchLocationError), #[error( "Materialization references the collection: '{collection_name}', which does not exist" diff --git a/src/catalog/extraction.rs b/crates/catalog/src/extraction.rs similarity index 98% rename from src/catalog/extraction.rs rename to crates/catalog/src/extraction.rs index 57df86870a..05cd736b9b 100644 --- a/src/catalog/extraction.rs +++ b/crates/catalog/src/extraction.rs @@ -57,12 +57,11 @@ impl Display for KeyError { #[cfg(test)] mod test { use super::*; - use crate::catalog::create; use itertools::Itertools; #[test] fn all_key_errors_are_returned() { - let db = create(":memory:").unwrap(); + let db = crate::create(":memory:").unwrap(); db.execute_batch( r##" diff --git a/src/catalog/inference.rs b/crates/catalog/src/inference.rs similarity index 94% rename from src/catalog/inference.rs rename to crates/catalog/src/inference.rs index cf495312de..6c6e43c8b2 100644 --- a/src/catalog/inference.rs +++ b/crates/catalog/src/inference.rs @@ -1,8 +1,6 @@ -use crate::catalog::{self, Schema, Scope, DB}; -use crate::doc::inference::Shape; -use crate::doc::Pointer; -use estuary_json::schema::types; -use estuary_json::Location; +use crate::{Schema, Scope, DB}; +use doc::{inference::Shape, Pointer}; +use json::{schema::types, Location}; use url::Url; /// Type information that's been inferred for a specific location within a schema. @@ -28,7 +26,7 @@ impl<'a> Inference<'a> { } /// Traverses the shape and persists inferences for all of the "projectable" locations within it. -pub fn register_all(scope: Scope, schema_uri: &Url) -> catalog::Result<()> { +pub fn register_all(scope: Scope, schema_uri: &Url) -> crate::Result<()> { let shape = Schema::shape_for(scope.db, scope.resource().id, schema_uri)?; for inference in get_inferences(&shape) { @@ -40,7 +38,7 @@ pub fn register_all(scope: Scope, schema_uri: &Url) -> catalog::Result<()> { /// Persists the given inference. This function is safe to call multiple times for the same /// inference. This behavior was chosen because inferences may be generated from both user-provided /// and automatically generated projections. -pub fn register_one(db: &DB, schema_uri: &str, inference: &Inference) -> catalog::Result<()> { +pub fn register_one(db: &DB, schema_uri: &str, inference: &Inference) -> crate::Result<()> { let types_json = inference.shape.type_.to_json_array(); let params = rusqlite::params![ schema_uri, @@ -131,12 +129,12 @@ fn usize_to_i64(unsigned: usize) -> i64 { #[cfg(test)] mod test { use super::*; - use estuary_json::schema::{build::build_schema, index::Index}; use itertools::Itertools; + use json::schema::{build::build_schema, index::Index}; #[test] fn inferences_are_registered_by_schema_uri() { - let db = catalog::create(":memory:").unwrap(); + let db = crate::create(":memory:").unwrap(); let schema = r##" type: object @@ -175,7 +173,7 @@ mod test { let scope = Scope::for_test(&db, 1); register_all(scope, &uri).expect("failed to register inferences"); - let actual = catalog::dump_table(&db, "inferences").unwrap(); + let actual = crate::dump_table(&db, "inferences").unwrap(); insta::assert_json_snapshot!(actual); } diff --git a/src/catalog/lambda.rs b/crates/catalog/src/lambda.rs similarity index 100% rename from src/catalog/lambda.rs rename to crates/catalog/src/lambda.rs diff --git a/src/catalog/mod.rs b/crates/catalog/src/lib.rs similarity index 98% rename from src/catalog/mod.rs rename to crates/catalog/src/lib.rs index c8448918a8..9674d360f5 100644 --- a/src/catalog/mod.rs +++ b/crates/catalog/src/lib.rs @@ -20,7 +20,6 @@ mod str_edit_distance; mod test_case; mod unicode_collation; -use crate::FLOW_VERSION; use std::convert::TryFrom; use std::path::Path; use url::Url; @@ -39,6 +38,8 @@ pub use selector::Selector; pub use source::Source; pub use test_case::TestCase; +pub static FLOW_VERSION: &'static str = env!("CARGO_PKG_VERSION"); + pub type Result = std::result::Result; /// Create a new and empty catalog database, returning an open connection. @@ -279,7 +280,7 @@ mod test { #[test] fn run_catalog_test() { let mut path = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); - path.extend(&["src", "catalog", "test_catalog.sh"]); + path.extend(&["src", "test_catalog.sh"]); let status = Command::new(path.as_os_str()) .spawn() diff --git a/src/catalog/materialization.rs b/crates/catalog/src/materialization.rs similarity index 89% rename from src/catalog/materialization.rs rename to crates/catalog/src/materialization.rs index c1ff6a7a67..a15d377a2d 100644 --- a/src/catalog/materialization.rs +++ b/crates/catalog/src/materialization.rs @@ -1,4 +1,4 @@ -use crate::catalog::{self, specs, Scope, DB}; +use crate::{specs, Scope, DB}; #[derive(Debug, Copy, Clone)] pub struct MaterializationTarget { @@ -10,7 +10,7 @@ impl MaterializationTarget { scope: &Scope, target_name: &str, spec: &specs::MaterializationTarget, - ) -> catalog::Result { + ) -> crate::Result { let uri = match &spec.config { specs::MaterializationConfig::Postgres(connection) => connection.uri.clone(), specs::MaterializationConfig::Sqlite(connection) => { @@ -44,7 +44,7 @@ impl MaterializationTarget { Ok(MaterializationTarget { id }) } - pub fn get_by_name(db: &DB, name: &str) -> catalog::Result { + pub fn get_by_name(db: &DB, name: &str) -> crate::Result { let mut stmt = db.prepare_cached( "SELECT target_id FROM materialization_targets WHERE target_name = ?;", )?; @@ -57,11 +57,11 @@ impl MaterializationTarget { #[cfg(test)] mod test { use super::*; - use crate::catalog::{self, Error}; + use crate::{create, Error}; #[test] fn get_by_name_returns_extant_target() { - let db = catalog::create(":memory:").unwrap(); + let db = create(":memory:").unwrap(); db.execute_batch( r##" insert into materialization_targets @@ -78,7 +78,7 @@ mod test { #[test] fn get_by_name_returns_err_when_named_target_does_not_exist() { - let db = catalog::create(":memory:").unwrap(); + let db = create(":memory:").unwrap(); let err = MaterializationTarget::get_by_name(&db, "nonExistant") .expect_err("expected an error from get_by_name"); assert!(matches!( diff --git a/src/catalog/nodejs/build.rs b/crates/catalog/src/nodejs/build.rs similarity index 97% rename from src/catalog/nodejs/build.rs rename to crates/catalog/src/nodejs/build.rs index 7b37aba1dd..1c9069d32a 100644 --- a/src/catalog/nodejs/build.rs +++ b/crates/catalog/src/nodejs/build.rs @@ -1,12 +1,14 @@ use super::typescript; -use crate::catalog::{self, ContentType, Error}; -use crate::doc::SchemaIndex; +use crate::{ContentType, Error, Resource, Schema}; +use doc::SchemaIndex; use include_dir::{include_dir, Dir}; use rusqlite::{params as sql_params, Connection as DB}; use serde::{Deserialize, Serialize}; use serde_json::Value; +use std::collections::hash_map::DefaultHasher; use std::collections::{BTreeMap, BTreeSet}; use std::fs; +use std::hash::{Hash, Hasher}; use std::io::Write; use std::iter::Iterator; use std::path; @@ -47,7 +49,7 @@ pub fn build_package(db: &DB, pkg: &path::Path) -> Result<(), Error> { let pack = fs::canonicalize(&pack)?; let pack = Url::from_file_path(&pack).unwrap(); - catalog::Resource::register(db, ContentType::NpmPack, &pack)?; + Resource::register(db, ContentType::NpmPack, &pack)?; log::info!("built NodeJS pack {:?}", pack); Ok(()) @@ -90,7 +92,7 @@ fn patch_package_json(db: &DB, pkg: &path::Path) -> Result<(), Error> { } fn generate_schemas_ts(db: &DB, pkg: &path::Path) -> Result<(), Error> { - let schemas = catalog::Schema::compile_all(db)?; + let schemas = Schema::compile_all(db)?; // Index all Schemas. let mut index = SchemaIndex::new(); @@ -332,7 +334,9 @@ fn ts_name(collection: &str, schema: &Url, is_alternate: bool) -> String { } } if is_alternate { - out = format!("{}_{:x}", out, fxhash::hash32(schema)); + let mut h = DefaultHasher::new(); + schema.hash(&mut h); + out = format!("{}_{:x}", out, h.finish()); } out } diff --git a/src/catalog/nodejs/mod.rs b/crates/catalog/src/nodejs/mod.rs similarity index 100% rename from src/catalog/nodejs/mod.rs rename to crates/catalog/src/nodejs/mod.rs diff --git a/src/catalog/nodejs/typescript/ast.rs b/crates/catalog/src/nodejs/typescript/ast.rs similarity index 100% rename from src/catalog/nodejs/typescript/ast.rs rename to crates/catalog/src/nodejs/typescript/ast.rs diff --git a/src/catalog/nodejs/typescript/mapping.rs b/crates/catalog/src/nodejs/typescript/mapping.rs similarity index 99% rename from src/catalog/nodejs/typescript/mapping.rs rename to crates/catalog/src/nodejs/typescript/mapping.rs index d41f2cff09..7735022e34 100644 --- a/src/catalog/nodejs/typescript/mapping.rs +++ b/crates/catalog/src/nodejs/typescript/mapping.rs @@ -1,9 +1,9 @@ use super::ast::{ASTProperty, ASTTuple, AST}; -use crate::doc::{ +use doc::{ inference::{ArrayShape, ObjShape, Provenance, Shape}, Schema, SchemaIndex, }; -use estuary_json::schema::types; +use json::schema::types; use std::collections::BTreeMap; pub struct Mapper<'a> { diff --git a/src/catalog/nodejs/typescript/mod.rs b/crates/catalog/src/nodejs/typescript/mod.rs similarity index 100% rename from src/catalog/nodejs/typescript/mod.rs rename to crates/catalog/src/nodejs/typescript/mod.rs diff --git a/src/catalog/projections.rs b/crates/catalog/src/projections.rs similarity index 96% rename from src/catalog/projections.rs rename to crates/catalog/src/projections.rs index 375965345d..b3ac19cace 100644 --- a/src/catalog/projections.rs +++ b/crates/catalog/src/projections.rs @@ -1,6 +1,5 @@ -use crate::catalog::inference; -use crate::catalog::{specs, Collection, Error, Result, Schema, Scope, DB}; -use crate::doc::inference::Shape; +use crate::{inference, specs, Collection, Error, Result, Schema, Scope, DB}; +use doc::inference::Shape; use rusqlite::params as sql_params; pub fn register_projections( @@ -127,7 +126,7 @@ fn contains_location(spec: &specs::Projections, location: &str, field: &str) -> #[cfg(test)] mod test { use super::*; - use crate::catalog::{create, Resource}; + use crate::{create, dump_tables, Resource}; use rusqlite::params as sql_params; use serde_json::json; @@ -266,8 +265,7 @@ mod test { register_projections(&scope, collection, &projections) .expect("failed to register projections"); - let actual = - crate::catalog::dump_tables(&db, &["projections", "partitions", "inferences"]).unwrap(); + let actual = dump_tables(&db, &["projections", "partitions", "inferences"]).unwrap(); insta::assert_json_snapshot!(actual); } diff --git a/src/catalog/regexp_sql_fn.rs b/crates/catalog/src/regexp_sql_fn.rs similarity index 67% rename from src/catalog/regexp_sql_fn.rs rename to crates/catalog/src/regexp_sql_fn.rs index 383e1549e1..055621ee84 100644 --- a/src/catalog/regexp_sql_fn.rs +++ b/crates/catalog/src/regexp_sql_fn.rs @@ -1,38 +1,26 @@ use regex::Regex; use rusqlite::{functions::FunctionFlags, Connection, Error, Result}; +use std::sync::Arc; + +type BoxError = Box; pub fn install(db: &Connection) -> Result<()> { + // See: https://github.com/rusqlite/rusqlite/blob/master/src/functions.rs let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC; db.create_scalar_function("regexp", 2, flags, move |ctx| { assert_eq!(ctx.len(), 2, "called with unexpected number of arguments"); - - let saved_re: Option<&Regex> = ctx.get_aux(0)?; - let new_re = match saved_re { - None => { - let s = ctx.get::(0)?; - match Regex::new(&s) { - Ok(r) => Some(r), - Err(err) => return Err(Error::UserFunctionError(Box::new(err))), - } - } - Some(_) => None, - }; - + let regexp: Arc = ctx.get_or_create_aux(0, |vr| -> Result<_, BoxError> { + Ok(Regex::new(vr.as_str()?)?) + })?; let is_match = { - let re = saved_re.unwrap_or_else(|| new_re.as_ref().unwrap()); - let text = ctx .get_raw(1) .as_str() .map_err(|e| Error::UserFunctionError(e.into()))?; - re.is_match(text) + regexp.is_match(text) }; - if let Some(re) = new_re { - ctx.set_aux(0, re); - } - Ok(is_match) }) } diff --git a/src/catalog/resource.rs b/crates/catalog/src/resource.rs similarity index 100% rename from src/catalog/resource.rs rename to crates/catalog/src/resource.rs diff --git a/src/catalog/schema.rs b/crates/catalog/src/schema.rs similarity index 99% rename from src/catalog/schema.rs rename to crates/catalog/src/schema.rs index 48c472c681..5187fdab7c 100644 --- a/src/catalog/schema.rs +++ b/crates/catalog/src/schema.rs @@ -1,6 +1,6 @@ use super::{specs, sql_params, ContentType, Error, Resource, Result, Scope, DB}; -use crate::doc::{inference, Schema as CompiledSchema, SchemaIndex}; -use estuary_json::schema::{build::build_schema, Application, Keyword}; +use doc::{inference, Schema as CompiledSchema, SchemaIndex}; +use json::schema::{build::build_schema, Application, Keyword}; use url::Url; pub const INLINE_POINTER_KEY: &str = "ptr"; diff --git a/src/catalog/scope.rs b/crates/catalog/src/scope.rs similarity index 99% rename from src/catalog/scope.rs rename to crates/catalog/src/scope.rs index c7f176909e..79c9b63314 100644 --- a/src/catalog/scope.rs +++ b/crates/catalog/src/scope.rs @@ -1,5 +1,5 @@ use super::{Error, Resource, Result, DB}; -use estuary_json::Location; +use json::Location; /// Scope holds context about the current catalog build process. It's used /// whenever the we need to know which Resource we're processing, diff --git a/src/catalog/selector.rs b/crates/catalog/src/selector.rs similarity index 100% rename from src/catalog/selector.rs rename to crates/catalog/src/selector.rs diff --git a/src/catalog/snapshots/estuary__catalog__derivation__test__register.snap b/crates/catalog/src/snapshots/catalog__derivation__test__register.snap similarity index 100% rename from src/catalog/snapshots/estuary__catalog__derivation__test__register.snap rename to crates/catalog/src/snapshots/catalog__derivation__test__register.snap diff --git a/src/catalog/snapshots/estuary__catalog__extraction__test__all_key_errors_are_returned.snap b/crates/catalog/src/snapshots/catalog__extraction__test__all_key_errors_are_returned.snap similarity index 100% rename from src/catalog/snapshots/estuary__catalog__extraction__test__all_key_errors_are_returned.snap rename to crates/catalog/src/snapshots/catalog__extraction__test__all_key_errors_are_returned.snap diff --git a/src/catalog/snapshots/estuary__catalog__inference__test__inferences_are_registered_by_schema_uri.snap b/crates/catalog/src/snapshots/catalog__inference__test__inferences_are_registered_by_schema_uri.snap similarity index 100% rename from src/catalog/snapshots/estuary__catalog__inference__test__inferences_are_registered_by_schema_uri.snap rename to crates/catalog/src/snapshots/catalog__inference__test__inferences_are_registered_by_schema_uri.snap diff --git a/src/catalog/snapshots/estuary__catalog__inference__test__inferences_are_returned_for_shape.snap b/crates/catalog/src/snapshots/catalog__inference__test__inferences_are_returned_for_shape.snap similarity index 100% rename from src/catalog/snapshots/estuary__catalog__inference__test__inferences_are_returned_for_shape.snap rename to crates/catalog/src/snapshots/catalog__inference__test__inferences_are_returned_for_shape.snap diff --git a/src/catalog/snapshots/estuary__catalog__projections__test__default_projections_and_inferences_are_registered.snap b/crates/catalog/src/snapshots/catalog__projections__test__default_projections_and_inferences_are_registered.snap similarity index 100% rename from src/catalog/snapshots/estuary__catalog__projections__test__default_projections_and_inferences_are_registered.snap rename to crates/catalog/src/snapshots/catalog__projections__test__default_projections_and_inferences_are_registered.snap diff --git a/src/catalog/source.rs b/crates/catalog/src/source.rs similarity index 100% rename from src/catalog/source.rs rename to crates/catalog/src/source.rs diff --git a/src/catalog/specs.rs b/crates/catalog/src/specs.rs similarity index 100% rename from src/catalog/specs.rs rename to crates/catalog/src/specs.rs diff --git a/src/catalog/str_edit_distance.rs b/crates/catalog/src/str_edit_distance.rs similarity index 100% rename from src/catalog/str_edit_distance.rs rename to crates/catalog/src/str_edit_distance.rs diff --git a/src/catalog/test_case.rs b/crates/catalog/src/test_case.rs similarity index 100% rename from src/catalog/test_case.rs rename to crates/catalog/src/test_case.rs diff --git a/src/catalog/test_catalog.sh b/crates/catalog/src/test_catalog.sh similarity index 100% rename from src/catalog/test_catalog.sh rename to crates/catalog/src/test_catalog.sh diff --git a/src/catalog/unicode_collation.rs b/crates/catalog/src/unicode_collation.rs similarity index 100% rename from src/catalog/unicode_collation.rs rename to crates/catalog/src/unicode_collation.rs diff --git a/tests/example_catalog_builds_test.rs b/crates/catalog/tests/example_catalog_builds_test.rs similarity index 88% rename from tests/example_catalog_builds_test.rs rename to crates/catalog/tests/example_catalog_builds_test.rs index 41ae0b76c2..f3ba547c1c 100644 --- a/tests/example_catalog_builds_test.rs +++ b/crates/catalog/tests/example_catalog_builds_test.rs @@ -1,6 +1,5 @@ -use estuary::catalog; -use estuary_protocol::flow::CollectionSpec; use pretty_env_logger; +use protocol::flow::CollectionSpec; use std::{env, path::PathBuf}; use url::Url; @@ -9,8 +8,9 @@ fn test_examples() { pretty_env_logger::init(); let root_dir = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); - let catalog_path = root_dir.join("examples/flow.yaml"); - let nodejs_dir = root_dir.join("target/nodejs/"); + let catalog_path = root_dir.join("../../examples/flow.yaml"); + let catalog_path = std::fs::canonicalize(catalog_path).unwrap(); + let nodejs_dir = root_dir.join("../../catalog-nodejs/"); let db = catalog::create(":memory:").unwrap(); let url = Url::from_file_path(&catalog_path).unwrap(); diff --git a/crates/derive/Cargo.toml b/crates/derive/Cargo.toml new file mode 100644 index 0000000000..0f0d64cdb3 --- /dev/null +++ b/crates/derive/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "derive" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +catalog = { path = "../catalog", version = "0.0.0" } +doc = { path = "../doc", version = "0.0.0" } +json = { path = "../json", version = "0.0.0" } +protocol = { path = "../protocol", version = "0.0.0" } +serve = { path = "../serve", version = "0.0.0" } + +futures = "*" +http = "*" +hyper = "*" +hyper-tls = "*" +hyperlocal = "*" +itertools = "*" +log = "*" +mime = "*" +pin-utils = "*" +prost = "*" +rocksdb = "*" +rusqlite = { version = "*", features = ["bundled", "collation", "column_decltype", "functions", "serde_json", "url"] } +serde = { version = "*", features = ["derive"] } +serde_json = { version = "*"} +thiserror = "*" +tokio = { version = "0.2", features = ["full"] } +tonic = { version = "*" } +url = "*" +uuid = "*" + +[dev-dependencies] +insta = "*" +tempfile = "*" +tower = "*" \ No newline at end of file diff --git a/src/derive/combine_api.rs b/crates/derive/src/combine_api.rs similarity index 98% rename from src/derive/combine_api.rs rename to crates/derive/src/combine_api.rs index 8fb8158627..ad0bd4efc2 100644 --- a/src/derive/combine_api.rs +++ b/crates/derive/src/combine_api.rs @@ -1,9 +1,9 @@ use super::combiner::Combiner; -use crate::doc::{reduce, Pointer, SchemaIndex}; -use estuary_protocol::flow; +use doc::{reduce, Pointer, SchemaIndex}; use futures::channel::mpsc; use futures::sink::SinkExt; use futures::stream::{Stream, StreamExt}; +use protocol::flow; use serde_json::Value; #[derive(thiserror::Error, Debug)] @@ -15,7 +15,7 @@ pub enum Error { #[error("parsing URL: {0:?}")] Url(#[from] url::ParseError), #[error("schema index: {0}")] - SchemaIndex(#[from] estuary_json::schema::index::Error), + SchemaIndex(#[from] json::schema::index::Error), #[error("JSON error: {0}")] Json(#[from] serde_json::Error), #[error("invalid arena range: {0:?}")] diff --git a/src/derive/combiner.rs b/crates/derive/src/combiner.rs similarity index 97% rename from src/derive/combiner.rs rename to crates/derive/src/combiner.rs index d22ced6814..9eb086703a 100644 --- a/src/derive/combiner.rs +++ b/crates/derive/src/combiner.rs @@ -1,5 +1,5 @@ -use crate::doc::{reduce, Pointer, SchemaIndex, Validator}; -use estuary_json::validator::FullContext; +use doc::{reduce, Pointer, SchemaIndex, Validator}; +use json::validator::FullContext; use serde_json::Value; use std::cmp::Ordering; use std::collections::BTreeSet; diff --git a/src/derive/context.rs b/crates/derive/src/context.rs similarity index 98% rename from src/derive/context.rs rename to crates/derive/src/context.rs index 5be8726ff4..54a02a10cb 100644 --- a/src/derive/context.rs +++ b/crates/derive/src/context.rs @@ -1,7 +1,7 @@ use super::lambda; use super::nodejs; -use crate::catalog::{self, sql_params}; -use crate::doc::{Pointer, SchemaIndex}; +use catalog::{self, sql_params}; +use doc::{Pointer, SchemaIndex}; use std::sync::Arc; use url::Url; diff --git a/src/derive/derive_api.rs b/crates/derive/src/derive_api.rs similarity index 99% rename from src/derive/derive_api.rs rename to crates/derive/src/derive_api.rs index efdf4fb09b..08d59ad1f8 100644 --- a/src/derive/derive_api.rs +++ b/crates/derive/src/derive_api.rs @@ -4,14 +4,14 @@ use super::context::{Context, Transform}; use super::lambda; use super::pipeline::PendingPipeline; use super::registers::{self, Registers}; -use crate::doc::{self, reduce}; -use estuary_json::validator; -use estuary_protocol::{consumer, flow, recoverylog}; +use doc::{self, reduce}; use futures::channel::mpsc; use futures::sink::SinkExt; use futures::stream::{Stream, StreamExt, TryStreamExt}; use futures::TryFutureExt; use itertools::izip; +use json::validator; +use protocol::{consumer, flow, recoverylog}; use std::default::Default; use std::fmt::Debug; use std::future::Future; @@ -24,7 +24,7 @@ pub enum Error { #[error("JSON error: {0}")] Json(#[from] serde_json::Error), #[error("Schema index: {0}")] - SchemaIndex(#[from] estuary_json::schema::index::Error), + SchemaIndex(#[from] json::schema::index::Error), #[error("channel send error: {0:?}")] SendError(#[from] mpsc::SendError), #[error("recv error from peer: {0}")] @@ -873,7 +873,7 @@ mod test { // Build and index the schema, leaking for `static lifetime. let schema_url = Url::parse("https://schema").unwrap(); let schema: doc::Schema = - estuary_json::schema::build::build_schema(schema_url.clone(), &schema).unwrap(); + json::schema::build::build_schema(schema_url.clone(), &schema).unwrap(); let schema = Box::leak(Box::new(schema)); let mut schema_index = doc::SchemaIndex::new(); diff --git a/src/derive/extract_api.rs b/crates/derive/src/extract_api.rs similarity index 96% rename from src/derive/extract_api.rs rename to crates/derive/src/extract_api.rs index 915887e5eb..69644e2911 100644 --- a/src/derive/extract_api.rs +++ b/crates/derive/src/extract_api.rs @@ -1,5 +1,6 @@ -use crate::doc::Pointer; -use estuary_protocol::flow; +use doc::Pointer; +use json::Number; +use protocol::flow; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -118,16 +119,16 @@ pub fn extract_field( serde_json::Value::Null => out.kind = flow::field::value::Kind::Null as i32, serde_json::Value::Bool(true) => out.kind = flow::field::value::Kind::True as i32, serde_json::Value::Bool(false) => out.kind = flow::field::value::Kind::False as i32, - serde_json::Value::Number(n) => match estuary_json::Number::from(n) { - estuary_json::Number::Float(d) => { + serde_json::Value::Number(n) => match Number::from(n) { + Number::Float(d) => { out.kind = flow::field::value::Kind::Double as i32; out.double = d; } - estuary_json::Number::Signed(s) => { + Number::Signed(s) => { out.kind = flow::field::value::Kind::Signed as i32; out.signed = s; } - estuary_json::Number::Unsigned(u) => { + Number::Unsigned(u) => { out.kind = flow::field::value::Kind::Unsigned as i32; out.unsigned = u; } diff --git a/src/derive/lambda.rs b/crates/derive/src/lambda.rs similarity index 98% rename from src/derive/lambda.rs rename to crates/derive/src/lambda.rs index fa366bdc21..84c3e191b0 100644 --- a/src/derive/lambda.rs +++ b/crates/derive/src/lambda.rs @@ -220,9 +220,9 @@ async fn check_headers( #[cfg(test)] pub mod test { use super::*; + use futures::channel::oneshot; use hyper::{Body, Request, Response}; use serde_json::{json, Value}; - use tokio::sync::oneshot; #[tokio::test] async fn test_hello_world() { @@ -321,7 +321,7 @@ pub mod test { let rx_stop = async move { rx_stop.await.unwrap(); }; - let server = crate::serve::unix_domain_socket(svc, &socket_path, rx_stop); + let server = serve::unix_domain_socket(svc, &socket_path, rx_stop); tokio::spawn(server); let client = hyper::Client::builder() diff --git a/src/derive/mod.rs b/crates/derive/src/lib.rs similarity index 95% rename from src/derive/mod.rs rename to crates/derive/src/lib.rs index 15d1557dd8..f615cbfdc7 100644 --- a/src/derive/mod.rs +++ b/crates/derive/src/lib.rs @@ -14,8 +14,8 @@ pub use extract_api::extract_uuid_parts; /// Common test utilities used by sub-modules. #[cfg(test)] pub mod test { - use crate::doc; - use estuary_protocol::flow; + use doc; + use protocol::flow; use serde_json::json; use url::Url; @@ -38,8 +38,7 @@ pub mod test { }); let uri = Url::parse("https://example/schema").unwrap(); - let scm: doc::Schema = - estuary_json::schema::build::build_schema(uri.clone(), &schema).unwrap(); + let scm: doc::Schema = json::schema::build::build_schema(uri.clone(), &schema).unwrap(); let scm = Box::leak(Box::new(scm)); let mut idx = doc::SchemaIndex::new(); diff --git a/src/derive/nodejs.rs b/crates/derive/src/nodejs.rs similarity index 94% rename from src/derive/nodejs.rs rename to crates/derive/src/nodejs.rs index 9efef8deb2..f24fe28cec 100644 --- a/src/derive/nodejs.rs +++ b/crates/derive/src/nodejs.rs @@ -1,5 +1,4 @@ use super::lambda; -use crate::catalog; use std::fs; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; @@ -24,13 +23,13 @@ pub struct NodeRuntime { impl NodeRuntime { /// Start a NodeJS worker using the NPM package extracted from the catalog database. - pub fn start(db: &catalog::DB, dir: impl AsRef) -> Result { + pub fn start(db: &rusqlite::Connection, dir: impl AsRef) -> Result { // Extract catalog pack.tgz to a new temp directory. let dir = dir.as_ref(); let pack_path = dir.join("npm-pack.tgz"); let pack_contents: Vec = db .prepare("SELECT content FROM resources WHERE content_type = ?")? - .query_row(catalog::sql_params![catalog::ContentType::NpmPack], |r| { + .query_row(rusqlite::params![catalog::ContentType::NpmPack], |r| { r.get(0) })?; fs::write(&pack_path, pack_contents)?; diff --git a/src/derive/pipeline.rs b/crates/derive/src/pipeline.rs similarity index 100% rename from src/derive/pipeline.rs rename to crates/derive/src/pipeline.rs diff --git a/src/derive/registers.rs b/crates/derive/src/registers.rs similarity index 98% rename from src/derive/registers.rs rename to crates/derive/src/registers.rs index 5ecd1accf0..af5e2a0563 100644 --- a/src/derive/registers.rs +++ b/crates/derive/src/registers.rs @@ -1,8 +1,8 @@ -use crate::doc::{reduce, SchemaIndex, Validator}; -use estuary_json::validator::FullContext; -use estuary_protocol::consumer::Checkpoint; +use doc::{reduce, SchemaIndex, Validator}; use futures::channel::oneshot; +use json::validator::FullContext; use prost::Message; +use protocol::consumer::Checkpoint; use serde_json::Value; use std::collections::HashMap; use url::Url; diff --git a/crates/doc/Cargo.toml b/crates/doc/Cargo.toml new file mode 100644 index 0000000000..f05001f5a2 --- /dev/null +++ b/crates/doc/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "doc" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +json = { path = "../json", version = "0.0.0" } + +itertools = "*" +lazy_static = "*" +regex = "*" +serde = { version = "*", features = ["derive"] } +serde_json = { version = "*", features = ["raw_value"] } +serde_yaml = "*" +thiserror = "*" +tinyvec = {version = "*", features = ["alloc"]} +url = "*" + +[dev-dependencies] +insta = "*" +quickcheck = "*" +quickcheck_macros = "*" \ No newline at end of file diff --git a/src/doc/annotation.rs b/crates/doc/src/annotation.rs similarity index 96% rename from src/doc/annotation.rs rename to crates/doc/src/annotation.rs index e48a3a06ac..3e6d387a3a 100644 --- a/src/doc/annotation.rs +++ b/crates/doc/src/annotation.rs @@ -1,5 +1,5 @@ use super::reduce; -use estuary_json::{schema, validator}; +use json::{schema, validator, Span}; use std::convert::TryFrom; /// Enumeration of JSON-Schema associated annotations understood by Estuary. @@ -49,7 +49,7 @@ impl schema::build::AnnotationBuilder for Annotation { /// Given outcomes of a successful validation, extract reduce annotations & hashes. pub fn extract_reduce_annotations<'a, C>( - span: estuary_json::Span, + span: Span, outcomes: &[(validator::Outcome<'a, Annotation>, C)], ) -> Vec<(&'a reduce::Strategy, u64)> where diff --git a/src/doc/diff.rs b/crates/doc/src/diff.rs similarity index 99% rename from src/doc/diff.rs rename to crates/doc/src/diff.rs index 495a09c3ff..c113a257a7 100644 --- a/src/doc/diff.rs +++ b/crates/doc/src/diff.rs @@ -1,8 +1,8 @@ -use estuary_json::Location; use itertools::{ EitherOrBoth::{Both, Left, Right}, Itertools, }; +use json::Location; use serde::Serialize; use serde_json::Value; diff --git a/src/doc/inference.rs b/crates/doc/src/inference.rs similarity index 99% rename from src/doc/inference.rs rename to crates/doc/src/inference.rs index 963688dff7..82d39cdcc8 100644 --- a/src/doc/inference.rs +++ b/crates/doc/src/inference.rs @@ -1,9 +1,10 @@ use super::{ptr::Token, reduce, Annotation, Pointer, Schema, SchemaIndex}; -use estuary_json::{ +use itertools::{self, EitherOrBoth, Itertools}; +use json::{ + json_cmp, schema::{types, Application, CoreAnnotation, Keyword, Validation}, LocatedProperty, Location, }; -use itertools::{self, EitherOrBoth, Itertools}; use lazy_static::lazy_static; use regex::Regex; use serde_json::Value; @@ -583,7 +584,7 @@ impl Shape { variants .iter() .map(|hl| hl.value.clone()) - .sorted_by(estuary_json::json_cmp) + .sorted_by(json_cmp) .collect::>(), ); } @@ -912,11 +913,12 @@ fn intersect_enum( Some(filter_enums_to_types(type_, l.into_iter()).collect()) } (Some(l), Some(r)) => { - let it = itertools::merge_join_by(l.into_iter(), r.into_iter(), estuary_json::json_cmp) - .filter_map(|eob| match eob { + let it = itertools::merge_join_by(l.into_iter(), r.into_iter(), json_cmp).filter_map( + |eob| match eob { EitherOrBoth::Both(l, _) => Some(l), _ => None, - }); + }, + ); let it = filter_enums_to_types(type_, it); Some(it.collect()) } @@ -930,7 +932,7 @@ fn union_enum(lhs: Option>, rhs: Option>) -> Option l, EitherOrBoth::Left(l) => l, @@ -1172,7 +1174,7 @@ impl Shape { #[cfg(test)] mod test { use super::{super::Annotation, *}; - use estuary_json::schema; + use json::schema; use serde_json::{json, Value}; use serde_yaml; diff --git a/src/doc/mod.rs b/crates/doc/src/lib.rs similarity index 53% rename from src/doc/mod.rs rename to crates/doc/src/lib.rs index 185cb2c8de..4ff6bcf225 100644 --- a/src/doc/mod.rs +++ b/crates/doc/src/lib.rs @@ -8,11 +8,11 @@ pub mod reduce; mod annotation; pub use annotation::{extract_reduce_annotations, Annotation}; -// Specialize estuary_json templates for the estuary `Annotation` type. -pub type Schema = estuary_json::schema::Schema; -pub type SchemaIndex<'sm> = estuary_json::schema::index::Index<'sm, Annotation>; -pub type Validator<'sm, C> = estuary_json::validator::Validator<'sm, Annotation, C>; -pub type FullContext = estuary_json::validator::FullContext; +// Specialize json templates for the estuary `Annotation` type. +pub type Schema = json::schema::Schema; +pub type SchemaIndex<'sm> = json::schema::index::Index<'sm, Annotation>; +pub type Validator<'sm, C> = json::validator::Validator<'sm, Annotation, C>; +pub type FullContext = json::validator::FullContext; mod diff; pub use diff::Diff; @@ -27,18 +27,18 @@ pub fn validate( val: &mut Validator, schema: &url::Url, doc: &serde_json::Value, -) -> Result +) -> Result where - C: estuary_json::validator::Context, + C: json::validator::Context, { val.prepare(schema) .expect("attempt to use invalid register schema URL for validation"); - let span = estuary_json::de::walk(doc, val).unwrap(); + let span = json::de::walk(doc, val).unwrap(); if val.invalid() { Err(FailedValidation { document: doc.clone(), - basic_output: estuary_json::validator::build_basic_output(val.outcomes()), + basic_output: json::validator::build_basic_output(val.outcomes()), }) } else { Ok(span) diff --git a/src/doc/ptr.rs b/crates/doc/src/ptr.rs similarity index 99% rename from src/doc/ptr.rs rename to crates/doc/src/ptr.rs index 061e3bfbf8..182979b569 100644 --- a/src/doc/ptr.rs +++ b/crates/doc/src/ptr.rs @@ -1,4 +1,4 @@ -use estuary_json::Location; +use json::{json_cmp, Location}; use serde_json as sj; use std::cmp::Ordering; use std::str::FromStr; @@ -35,8 +35,8 @@ impl Pointer { /// always use the raw property names without performing any conversions. /// /// ``` - /// use estuary_json::Location; - /// use estuary::doc::ptr::{Pointer, Token}; + /// use json::Location; + /// use doc::ptr::{Pointer, Token}; /// /// let root = Location::Root; /// let foo = root.push_prop("foo"); @@ -211,7 +211,7 @@ impl Pointer { pub fn compare(ptrs: &[Self], lhs: &sj::Value, rhs: &sj::Value) -> Ordering { ptrs.iter() .map(|ptr| { - estuary_json::json_cmp( + json_cmp( ptr.query(lhs).unwrap_or(&sj::Value::Null), ptr.query(rhs).unwrap_or(&sj::Value::Null), ) diff --git a/src/doc/reduce/mod.rs b/crates/doc/src/reduce/mod.rs similarity index 97% rename from src/doc/reduce/mod.rs rename to crates/doc/src/reduce/mod.rs index f1abd4fd41..0bfe40dc77 100644 --- a/src/doc/reduce/mod.rs +++ b/crates/doc/src/reduce/mod.rs @@ -1,6 +1,6 @@ pub use super::{extract_reduce_annotations, validate, FailedValidation, FullContext, Validator}; -pub use estuary_json::{schema::types, validator::Context, LocatedItem, LocatedProperty, Location}; use itertools::EitherOrBoth; +pub use json::{schema::types, validator::Context, LocatedItem, LocatedProperty, Location}; use serde_json::Value; use url::Url; @@ -219,8 +219,8 @@ fn count_nodes(v: &Value) -> usize { pub mod test { use super::*; - use crate::doc::{FullContext, Schema, SchemaIndex, Validator}; - use estuary_json::schema::build::build_schema; + use crate::{FullContext, Schema, SchemaIndex, Validator}; + use json::schema::build::build_schema; pub use serde_json::{json, Value}; use std::error::Error as StdError; diff --git a/src/doc/reduce/set.rs b/crates/doc/src/reduce/set.rs similarity index 99% rename from src/doc/reduce/set.rs rename to crates/doc/src/reduce/set.rs index 786e2d5917..d70adfa42f 100644 --- a/src/doc/reduce/set.rs +++ b/crates/doc/src/reduce/set.rs @@ -1,6 +1,6 @@ use super::{reduce_item, reduce_prop, Cursor, Error, Index, Location, Reducer, Result}; -use estuary_json::json_cmp_at; use itertools::EitherOrBoth; +use json::json_cmp_at; use serde_json::{Map, Value}; use super::strategy::Set; diff --git a/src/doc/reduce/strategy.rs b/crates/doc/src/reduce/strategy.rs similarity index 99% rename from src/doc/reduce/strategy.rs rename to crates/doc/src/reduce/strategy.rs index 30b4829885..9f9253e94d 100644 --- a/src/doc/reduce/strategy.rs +++ b/crates/doc/src/reduce/strategy.rs @@ -1,6 +1,6 @@ use super::{count_nodes, reduce_item, reduce_prop, Cursor, Error, Reducer, Result}; -use estuary_json::{json_cmp, json_cmp_at}; use itertools::EitherOrBoth; +use json::{json_cmp, json_cmp_at, Number}; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::cmp::Ordering; @@ -248,7 +248,7 @@ impl Strategy { } => { *tape = &tape[1..]; - let sum = estuary_json::Number::checked_add((&lhs).into(), (&rhs).into()); + let sum = Number::checked_add((&lhs).into(), (&rhs).into()); let sum = sum .ok_or(Error::SumNumericOverflow) .map_err(|err| Error::at(loc, err))?; diff --git a/src/doc/varint.rs b/crates/doc/src/varint.rs similarity index 100% rename from src/doc/varint.rs rename to crates/doc/src/varint.rs diff --git a/tests/reduce_annotations_test.rs b/crates/doc/tests/reduce_annotations_test.rs similarity index 98% rename from tests/reduce_annotations_test.rs rename to crates/doc/tests/reduce_annotations_test.rs index 7e320f7026..428fc764af 100644 --- a/tests/reduce_annotations_test.rs +++ b/crates/doc/tests/reduce_annotations_test.rs @@ -4,9 +4,9 @@ extern crate quickcheck; #[macro_use(quickcheck)] extern crate quickcheck_macros; -use estuary::doc::{reduce, FullContext, Schema, SchemaIndex, Validator}; -use estuary_json::{schema::build::build_schema, validator::Context}; +use doc::{reduce, FullContext, Schema, SchemaIndex, Validator}; use itertools::{EitherOrBoth, Itertools}; +use json::{schema::build::build_schema, validator::Context}; use serde::Deserialize; use serde_json::{json, Map, Value}; use std::collections::BTreeMap; diff --git a/crates/flow-worker/Cargo.toml b/crates/flow-worker/Cargo.toml new file mode 100644 index 0000000000..1c5fd5d0f9 --- /dev/null +++ b/crates/flow-worker/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "flow-worker" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +catalog = { path = "../catalog", version = "0.0.0" } +derive = { path = "../derive", version = "0.0.0" } +doc = { path = "../doc", version = "0.0.0" } +protocol = { path = "../protocol", version = "0.0.0" } +serve = { path = "../serve", version = "0.0.0" } + +anyhow = "*" +futures = "*" +hyper = "*" +libc = "*" +log = "*" +pretty_env_logger = "*" +rocksdb = "*" +structopt = "*" +tokio = { version = "0.2", features = ["full"] } +tower = "*" \ No newline at end of file diff --git a/build.rs b/crates/flow-worker/build.rs similarity index 100% rename from build.rs rename to crates/flow-worker/build.rs diff --git a/cmd/flow-worker/main.rs b/crates/flow-worker/src/main.rs similarity index 90% rename from cmd/flow-worker/main.rs rename to crates/flow-worker/src/main.rs index 8a07edbfce..119437decb 100644 --- a/cmd/flow-worker/main.rs +++ b/crates/flow-worker/src/main.rs @@ -1,10 +1,6 @@ use anyhow::{Context, Error}; -use estuary::{ - catalog::{self}, - derive, doc, -}; -use estuary_protocol::flow; use futures::{select, FutureExt}; +use protocol::flow; use std::path::PathBuf; use std::sync::Arc; use structopt::StructOpt; @@ -91,11 +87,8 @@ impl ExtractCommand { tower::service_fn(move |req: hyper::Request| extract_api.call(req)); // Bind local listener and begin serving. - let server = estuary::serve::unix_domain_socket( - service, - &self.grpc_socket_path, - register_signal_handlers()?, - ); + let server = + serve::unix_domain_socket(service, &self.grpc_socket_path, register_signal_handlers()?); let server_handle = tokio::spawn(server); // Signal to host process that we're ready to accept connections. @@ -118,11 +111,8 @@ impl CombineCommand { tower::service_fn(move |req: hyper::Request| combine_api.call(req)); // Bind local listener and begin serving. - let server = estuary::serve::unix_domain_socket( - service, - &self.grpc_socket_path, - register_signal_handlers()?, - ); + let server = + serve::unix_domain_socket(service, &self.grpc_socket_path, register_signal_handlers()?); let server_handle = tokio::spawn(server); // Signal to host process that we're ready to accept connections. @@ -164,7 +154,7 @@ impl DeriveCommand { &self.dir, [ rocksdb::DEFAULT_COLUMN_FAMILY_NAME, - crate::derive::registers::REGISTERS_CF, + derive::registers::REGISTERS_CF, ] .iter(), ) @@ -185,11 +175,8 @@ impl DeriveCommand { tower::service_fn(move |req: hyper::Request| derive_api.call(req)); // Bind local listener and begin serving. - let server = estuary::serve::unix_domain_socket( - service, - &self.grpc_socket_path, - register_signal_handlers()?, - ); + let server = + serve::unix_domain_socket(service, &self.grpc_socket_path, register_signal_handlers()?); let server_handle = tokio::spawn(server); // Invoke any user-provide runtime bootstraps. @@ -215,9 +202,7 @@ fn register_signal_handlers() -> Result, E }) } -fn build_schema_index( - db: &rusqlite::Connection, -) -> Result<&'static doc::SchemaIndex<'static>, Error> { +fn build_schema_index(db: &catalog::DB) -> Result<&'static doc::SchemaIndex<'static>, Error> { // Compile the bundle of catalog schemas. Then, deliberately "leak" the // immutable Schema bundle for the remainder of program in order to achieve // a 'static lifetime, which is required for use in spawned tokio Tasks (and diff --git a/crates/flowctl/Cargo.toml b/crates/flowctl/Cargo.toml new file mode 100644 index 0000000000..5154f72f08 --- /dev/null +++ b/crates/flowctl/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "flowctl" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +catalog = { path = "../catalog", version = "0.0.0" } +doc = { path = "../doc", version = "0.0.0" } +materialization = { path = "../materialization", version = "0.0.0" } +runtime = { path = "../runtime", version = "0.0.0" } +testing = { path = "../testing", version = "0.0.0" } + +anyhow = "*" +atty = "*" +futures = "*" +itertools = "*" +log = "*" +pretty_env_logger = "*" +rusqlite = { version = "*", features = ["bundled", "collation", "column_decltype", "functions", "serde_json", "url"] } +schemars = "*" +serde_json = { version = "*"} +structopt = "*" +tempfile = "*" +tokio = { version = "0.2", features = ["full"] } +url = "*" + +[dev-dependencies] +lazy_static = "*" +tempfile = "*" \ No newline at end of file diff --git a/crates/flowctl/build.rs b/crates/flowctl/build.rs new file mode 100644 index 0000000000..b0ca865bea --- /dev/null +++ b/crates/flowctl/build.rs @@ -0,0 +1,13 @@ +// The purpose of this is to use the value of the `FLOW_VERSION` variable in order to set +// `CARGO_PKG_VERSION`. `CARGO_PKG_VERSION` is the standard env variable that cargo always sets +// to the current version. Normally that version is taken from the Cargo.toml file, but since +// we want to determine the version dynamically, we need to set this variable manually. +// +// Note that it does not work to set/export `CARGO_PKG_VERSION` in any other way besides in this +// build script. Cargo will simply overwrite the variable, unless it's set by the build script of +// the current crate. +fn main() { + if let Ok(vers) = std::env::var("FLOW_VERSION") { + println!("cargo:rustc-env=CARGO_PKG_VERSION={}", vers); + } +} diff --git a/cmd/flowctl/main.rs b/crates/flowctl/src/main.rs similarity index 97% rename from cmd/flowctl/main.rs rename to crates/flowctl/src/main.rs index 33533a5654..c804570587 100644 --- a/cmd/flowctl/main.rs +++ b/crates/flowctl/src/main.rs @@ -1,5 +1,4 @@ use anyhow::{Context, Error}; -use estuary::{catalog, doc, materialization, runtime, testing}; use futures::FutureExt; use itertools::Itertools; use std::convert::TryFrom; @@ -160,11 +159,11 @@ fn init_logging(args: &Args) { } else { match verbosity { i32::MIN..=-2 => "off", - -1 => "off,estuary=warn,flowctl=warn", - 0 => "error,estuary=warn,flowctl=info", - 1 => "warn,estuary=info,flowctl=info", - 2 => "warn,estuary=debug,flowctl=debug", - 3..=i32::MAX => "info,estuary=trace,flowctl=trace", + -1 => "off,warn,flowctl=warn", + 0 => "warn,flowctl=info", + 1 => "info", + 2 => "debug", + 3..=i32::MAX => "trace", } }; builder.parse_filters(log_filters); @@ -300,8 +299,7 @@ async fn exec_external_command(command: &[String]) -> Result<(), Error> { for arg in command.iter().skip(1) { cmd.arg(arg); } - let child = cmd.spawn()?; - let exit_status = child.await?; + let exit_status = cmd.spawn()?.await?; if exit_status.success() { Ok(()) } else { @@ -518,7 +516,7 @@ async fn do_test(args: TestArgs) -> Result<(), Error> { fn do_dump_schema() -> Result<(), Error> { let settings = schemars::gen::SchemaSettings::draft07(); let gen = schemars::gen::SchemaGenerator::new(settings); - let schema = gen.into_root_schema_for::(); + let schema = gen.into_root_schema_for::(); serde_json::to_writer_pretty(std::io::stdout(), &schema)?; Ok(()) diff --git a/tests/materialize_tests.rs b/crates/flowctl/tests/materialize_tests.rs similarity index 94% rename from tests/materialize_tests.rs rename to crates/flowctl/tests/materialize_tests.rs index fadbfa22db..df5dd9227d 100644 --- a/tests/materialize_tests.rs +++ b/crates/flowctl/tests/materialize_tests.rs @@ -1,7 +1,8 @@ use std::process::{Command, Output, Stdio}; const FLOWCTL: &str = env!("CARGO_BIN_EXE_flowctl"); -const EXAMPLE_CATALOG: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/examples/flow.yaml"); +const EXAMPLE_CATALOG: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../examples/flow.yaml"); +const NODEJS_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../catalog-nodejs"); // We'll build the catalog once, and then re-use the same result for all materialization tests. // This is just to save time, since the npm portion of catalog builds can take a while. @@ -26,7 +27,13 @@ fn build_example_catalog() -> Setup { let catalog_path = tempfile.as_ref().display().to_string(); let build_output = run_flowctl( catalog_path.as_str(), - &["build", "--source", EXAMPLE_CATALOG], + &[ + "build", + "--source", + EXAMPLE_CATALOG, + "--nodejs-dir", + NODEJS_DIR, + ], ); assert_eq!( 0, build_output.exit_code, diff --git a/crates/json/Cargo.toml b/crates/json/Cargo.toml new file mode 100644 index 0000000000..ae0bc1bfb3 --- /dev/null +++ b/crates/json/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "json" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +fxhash = "*" +itertools = "*" +percent-encoding = "*" +regex = "*" +serde = "*" +serde_json = "*" +serde_yaml = "*" +thiserror = "*" +tinyvec = {version = "*", features = ["alloc"]} +url = "*" + +[dev-dependencies] +glob = "*" diff --git a/json/compare.rs b/crates/json/src/compare.rs similarity index 100% rename from json/compare.rs rename to crates/json/src/compare.rs diff --git a/json/de.rs b/crates/json/src/de.rs similarity index 100% rename from json/de.rs rename to crates/json/src/de.rs diff --git a/json/lib.rs b/crates/json/src/lib.rs similarity index 99% rename from json/lib.rs rename to crates/json/src/lib.rs index 1d082307f9..52fc229a3b 100644 --- a/json/lib.rs +++ b/crates/json/src/lib.rs @@ -39,7 +39,7 @@ impl Span { /// /// Examples: /// ``` -/// use estuary_json::Location; +/// use json::Location; /// /// let l0 = Location::Root; /// let l1 = l0.push_prop("foo"); diff --git a/json/number.rs b/crates/json/src/number.rs similarity index 100% rename from json/number.rs rename to crates/json/src/number.rs diff --git a/json/schema/build.rs b/crates/json/src/schema/build.rs similarity index 100% rename from json/schema/build.rs rename to crates/json/src/schema/build.rs diff --git a/json/schema/index.rs b/crates/json/src/schema/index.rs similarity index 100% rename from json/schema/index.rs rename to crates/json/src/schema/index.rs diff --git a/json/schema/intern.rs b/crates/json/src/schema/intern.rs similarity index 100% rename from json/schema/intern.rs rename to crates/json/src/schema/intern.rs diff --git a/json/schema/keywords.rs b/crates/json/src/schema/keywords.rs similarity index 100% rename from json/schema/keywords.rs rename to crates/json/src/schema/keywords.rs diff --git a/json/schema/mod.rs b/crates/json/src/schema/mod.rs similarity index 100% rename from json/schema/mod.rs rename to crates/json/src/schema/mod.rs diff --git a/json/schema/types.rs b/crates/json/src/schema/types.rs similarity index 98% rename from json/schema/types.rs rename to crates/json/src/schema/types.rs index 17634cb921..3fd7392568 100644 --- a/json/schema/types.rs +++ b/crates/json/src/schema/types.rs @@ -117,7 +117,7 @@ impl Set { /// Returns an iterator over the type names as static strings. /// /// ``` - /// use estuary_json::schema::types::*; + /// use json::schema::types::*; /// /// let ty = ARRAY | OBJECT | NULL; /// @@ -138,7 +138,7 @@ impl Set { /// Returns a vec containing owned strings representing the types in this set. /// /// ``` - /// use estuary_json::schema::types::*; + /// use json::schema::types::*; /// /// let ty = ARRAY | OBJECT | NULL; /// @@ -157,7 +157,7 @@ impl Set { /// Returns the `Set` value for a single type with the given name. /// /// ``` - /// use estuary_json::schema::types::*; + /// use json::schema::types::*; /// assert_eq!(Some(FRACTIONAL | INTEGER), Set::for_type_name("number")); /// assert_eq!(Some(FRACTIONAL), Set::for_type_name("fractional")); /// assert_eq!(Some(INTEGER), Set::for_type_name("integer")); @@ -208,7 +208,7 @@ impl Set { /// Returns true if this Set represents exactly one scalar type besides null. /// /// ``` - /// use estuary_json::schema::types::*; + /// use json::schema::types::*; /// /// assert!(STRING.is_single_scalar_type()); /// assert!(INTEGER.is_single_scalar_type()); diff --git a/json/validator.rs b/crates/json/src/validator.rs similarity index 100% rename from json/validator.rs rename to crates/json/src/validator.rs diff --git a/json/tests/de_test.rs b/crates/json/tests/de_test.rs similarity index 97% rename from json/tests/de_test.rs rename to crates/json/tests/de_test.rs index 400365e4b2..6c31cc68d1 100644 --- a/json/tests/de_test.rs +++ b/crates/json/tests/de_test.rs @@ -1,4 +1,4 @@ -use estuary_json::{de::walk, LocatedItem, LocatedProperty, Location, Number, Span, Walker}; +use json::{de::walk, LocatedItem, LocatedProperty, Location, Number, Span, Walker}; use serde_json::{json, Deserializer}; #[test] diff --git a/json/tests/document_test.rs b/crates/json/tests/document_test.rs similarity index 93% rename from json/tests/document_test.rs rename to crates/json/tests/document_test.rs index 182b42c386..0d5e80584c 100644 --- a/json/tests/document_test.rs +++ b/crates/json/tests/document_test.rs @@ -1,4 +1,4 @@ -use estuary_json::Location; +use json::Location; #[test] fn test_formatting() { diff --git a/json/tests/draft2019_tests.rs b/crates/json/tests/draft2019_tests.rs similarity index 100% rename from json/tests/draft2019_tests.rs rename to crates/json/tests/draft2019_tests.rs diff --git a/json/tests/official b/crates/json/tests/official similarity index 100% rename from json/tests/official rename to crates/json/tests/official diff --git a/json/tests/regenerate-tests.sh b/crates/json/tests/regenerate-tests.sh similarity index 100% rename from json/tests/regenerate-tests.sh rename to crates/json/tests/regenerate-tests.sh diff --git a/json/tests/schema-fixtures/2019-09-meta-applicator.json b/crates/json/tests/schema-fixtures/2019-09-meta-applicator.json similarity index 100% rename from json/tests/schema-fixtures/2019-09-meta-applicator.json rename to crates/json/tests/schema-fixtures/2019-09-meta-applicator.json diff --git a/json/tests/schema-fixtures/2019-09-meta-content.json b/crates/json/tests/schema-fixtures/2019-09-meta-content.json similarity index 100% rename from json/tests/schema-fixtures/2019-09-meta-content.json rename to crates/json/tests/schema-fixtures/2019-09-meta-content.json diff --git a/json/tests/schema-fixtures/2019-09-meta-core.json b/crates/json/tests/schema-fixtures/2019-09-meta-core.json similarity index 100% rename from json/tests/schema-fixtures/2019-09-meta-core.json rename to crates/json/tests/schema-fixtures/2019-09-meta-core.json diff --git a/json/tests/schema-fixtures/2019-09-meta-data.json b/crates/json/tests/schema-fixtures/2019-09-meta-data.json similarity index 100% rename from json/tests/schema-fixtures/2019-09-meta-data.json rename to crates/json/tests/schema-fixtures/2019-09-meta-data.json diff --git a/json/tests/schema-fixtures/2019-09-meta-format.json b/crates/json/tests/schema-fixtures/2019-09-meta-format.json similarity index 100% rename from json/tests/schema-fixtures/2019-09-meta-format.json rename to crates/json/tests/schema-fixtures/2019-09-meta-format.json diff --git a/json/tests/schema-fixtures/2019-09-meta-validation.json b/crates/json/tests/schema-fixtures/2019-09-meta-validation.json similarity index 100% rename from json/tests/schema-fixtures/2019-09-meta-validation.json rename to crates/json/tests/schema-fixtures/2019-09-meta-validation.json diff --git a/json/tests/schema-fixtures/2019-09-schema.json b/crates/json/tests/schema-fixtures/2019-09-schema.json similarity index 100% rename from json/tests/schema-fixtures/2019-09-schema.json rename to crates/json/tests/schema-fixtures/2019-09-schema.json diff --git a/json/tests/schema_test.rs b/crates/json/tests/schema_test.rs similarity index 95% rename from json/tests/schema_test.rs rename to crates/json/tests/schema_test.rs index 66b59a824d..02df076c4e 100644 --- a/json/tests/schema_test.rs +++ b/crates/json/tests/schema_test.rs @@ -1,4 +1,4 @@ -use estuary_json::schema::{build, index, CoreAnnotation}; +use json::schema::{build, index, CoreAnnotation}; use serde_json::json; type Result = std::result::Result<(), Box>; diff --git a/json/tests/validator_test_utils.rs b/crates/json/tests/validator_test_utils.rs similarity index 99% rename from json/tests/validator_test_utils.rs rename to crates/json/tests/validator_test_utils.rs index 0d34352593..a2189afd2f 100644 --- a/json/tests/validator_test_utils.rs +++ b/crates/json/tests/validator_test_utils.rs @@ -1,10 +1,10 @@ //! Test utilities that are used by the automaically generated validation tests. -use estuary_json::{ +use glob; +use json::{ de, schema::{build, index, CoreAnnotation, Schema}, validator, }; -use glob; use serde_json as sj; use std::{env, fs, io, path}; diff --git a/crates/labels/Cargo.toml b/crates/labels/Cargo.toml new file mode 100644 index 0000000000..f0f808175c --- /dev/null +++ b/crates/labels/Cargo.toml @@ -0,0 +1,5 @@ +[package] +name = "labels" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" \ No newline at end of file diff --git a/src/labels.rs b/crates/labels/src/lib.rs similarity index 86% rename from src/labels.rs rename to crates/labels/src/lib.rs index 460b0a87c9..3ad6bd037f 100644 --- a/src/labels.rs +++ b/crates/labels/src/lib.rs @@ -24,14 +24,14 @@ pub mod values { #[macro_export] macro_rules! label_set { ($($key:expr => $value:expr),* $(,)*) => {{ - let mut labels: Vec = Vec::new(); + let mut labels: Vec = Vec::new(); $( - labels.push(estuary_protocol::protocol::Label { + labels.push(protocol::protocol::Label { name: String::from($key), value: String::from($value), }); )* labels.sort_by(|a, b| a.name.cmp(&b.name)); - estuary_protocol::protocol::LabelSet { labels } + protocol::protocol::LabelSet { labels } }} } diff --git a/crates/materialization/Cargo.toml b/crates/materialization/Cargo.toml new file mode 100644 index 0000000000..7e2fdc0d88 --- /dev/null +++ b/crates/materialization/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "materialization" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +catalog = { path = "../catalog", version = "0.0.0" } +doc = { path = "../doc", version = "0.0.0" } +json = { path = "../json", version = "0.0.0" } +labels = { path = "../labels", version = "0.0.0" } +protocol = { path = "../protocol", version = "0.0.0" } + +itertools = "*" +log = "*" +prost-types = "*" +rusqlite = { version = "*", features = ["bundled", "collation", "column_decltype", "functions", "serde_json", "url"] } +serde = { version = "*", features = ["derive"] } +serde_json = { version = "*"} +skim = "*" +thiserror = "*" +tuikit = "*" +unicode-width = "*" + +[dev-dependencies] +insta = "*" \ No newline at end of file diff --git a/src/materialization/error.rs b/crates/materialization/src/error.rs similarity index 98% rename from src/materialization/error.rs rename to crates/materialization/src/error.rs index 9f8614aee5..03705c7dd2 100644 --- a/src/materialization/error.rs +++ b/crates/materialization/src/error.rs @@ -1,6 +1,5 @@ -use crate::catalog; -use estuary_protocol::flow::Projection; use itertools::Itertools; +use protocol::flow::Projection; use std::collections::BTreeMap; use std::fmt; diff --git a/src/materialization/mod.rs b/crates/materialization/src/lib.rs similarity index 98% rename from src/materialization/mod.rs rename to crates/materialization/src/lib.rs index b67b2777ae..0434342a26 100644 --- a/src/materialization/mod.rs +++ b/crates/materialization/src/lib.rs @@ -5,11 +5,10 @@ mod sql; pub use self::error::{Error, NaughtyProjections}; use self::selection_ui::interactive_select_projections; use self::sql::SqlMaterializationConfig; -use crate::catalog::{self, DB}; -use crate::label_set; -use estuary_json::schema::types; -use estuary_protocol::consumer; -use estuary_protocol::flow::{CollectionSpec, Inference, Projection}; +use json::schema::types; +use protocol::consumer; +use protocol::flow::{CollectionSpec, Inference, Projection}; +use rusqlite::Connection as DB; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fmt::Display; @@ -58,7 +57,7 @@ fn create_shard_spec( target_name: &str, table_name: &str, ) -> consumer::ShardSpec { - use crate::labels::{keys, values}; + use labels::{keys, label_set, values}; let id = materialization_shard_id( collection_name, target_name, @@ -412,6 +411,7 @@ pub struct PayloadGenerationParameters<'a> { #[cfg(test)] mod test { use super::*; + use protocol::flow::inference; #[test] fn collection_spec_is_looked_up_by_name() { @@ -576,8 +576,6 @@ mod test { user_provided: bool, is_primary_key: bool, ) -> Projection { - use estuary_protocol::flow::inference; - let string = if types.overlaps(types::STRING) { Some(inference::String { content_type: String::new(), diff --git a/src/materialization/selection_ui.rs b/crates/materialization/src/selection_ui.rs similarity index 99% rename from src/materialization/selection_ui.rs rename to crates/materialization/src/selection_ui.rs index fad1aef0a8..3f004fb239 100644 --- a/src/materialization/selection_ui.rs +++ b/crates/materialization/src/selection_ui.rs @@ -1,6 +1,6 @@ -use crate::materialization::Error; -use estuary_protocol::flow::{CollectionSpec, Projection}; +use crate::Error; use itertools::Itertools; +use protocol::flow::{CollectionSpec, Projection}; use skim::{ AnsiString, DisplayContext, ItemPreview, PreviewContext, Selector, Skim, SkimItem, SkimOptions, }; diff --git a/src/materialization/snapshots/estuary__materialization__sql__test__ddl_is_generated_for_posgres_with_composit_primary_keys.snap b/crates/materialization/src/snapshots/materialization__sql__test__ddl_is_generated_for_posgres_with_composit_primary_keys.snap similarity index 100% rename from src/materialization/snapshots/estuary__materialization__sql__test__ddl_is_generated_for_posgres_with_composit_primary_keys.snap rename to crates/materialization/src/snapshots/materialization__sql__test__ddl_is_generated_for_posgres_with_composit_primary_keys.snap diff --git a/src/materialization/snapshots/estuary__materialization__sql__test__ddl_is_generated_for_sqlite_with_composit_primary_keys.snap b/crates/materialization/src/snapshots/materialization__sql__test__ddl_is_generated_for_sqlite_with_composit_primary_keys.snap similarity index 100% rename from src/materialization/snapshots/estuary__materialization__sql__test__ddl_is_generated_for_sqlite_with_composit_primary_keys.snap rename to crates/materialization/src/snapshots/materialization__sql__test__ddl_is_generated_for_sqlite_with_composit_primary_keys.snap diff --git a/src/materialization/snapshots/estuary__materialization__sql__test__ddl_is_generated_without_any_primary_key.snap b/crates/materialization/src/snapshots/materialization__sql__test__ddl_is_generated_without_any_primary_key.snap similarity index 100% rename from src/materialization/snapshots/estuary__materialization__sql__test__ddl_is_generated_without_any_primary_key.snap rename to crates/materialization/src/snapshots/materialization__sql__test__ddl_is_generated_without_any_primary_key.snap diff --git a/src/materialization/snapshots/estuary__materialization__sql__test__invalid_projections_are_returned_in_a_single_error.snap b/crates/materialization/src/snapshots/materialization__sql__test__invalid_projections_are_returned_in_a_single_error.snap similarity index 100% rename from src/materialization/snapshots/estuary__materialization__sql__test__invalid_projections_are_returned_in_a_single_error.snap rename to crates/materialization/src/snapshots/materialization__sql__test__invalid_projections_are_returned_in_a_single_error.snap diff --git a/src/materialization/sql.rs b/crates/materialization/src/sql.rs similarity index 99% rename from src/materialization/sql.rs rename to crates/materialization/src/sql.rs index 67177cf8b3..f669061a9d 100644 --- a/src/materialization/sql.rs +++ b/crates/materialization/src/sql.rs @@ -1,6 +1,6 @@ use super::{NaughtyProjections, PayloadGenerationParameters, Projection}; -use estuary_json::schema::types; use itertools::Itertools; +use json::schema::types; use serde::{Deserialize, Serialize}; use std::fmt::{self, Write}; @@ -580,9 +580,9 @@ impl<'a> fmt::Display for TableDescription<'a> { #[cfg(test)] mod test { use super::*; - use crate::materialization::flow_document_projection; - use estuary_protocol::flow::{inference, CollectionSpec, Inference}; + use crate::flow_document_projection; use insta::assert_snapshot; + use protocol::flow::{inference, CollectionSpec, Inference}; #[test] fn ddl_is_generated_without_any_primary_key() { diff --git a/crates/protocol/Cargo.toml b/crates/protocol/Cargo.toml new file mode 100644 index 0000000000..3ece98adb7 --- /dev/null +++ b/crates/protocol/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "protocol" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +humantime-serde = "*" +prost = "*" +prost-types = "*" +serde = "*" +tonic = "*" + +[dev-dependencies] +insta = "*" +serde_json = "*" + +[build-dependencies] +tonic-build = "*" diff --git a/protocol-rs/build.rs b/crates/protocol/build.rs similarity index 100% rename from protocol-rs/build.rs rename to crates/protocol/build.rs diff --git a/protocol-rs/src/consumer.rs b/crates/protocol/src/consumer.rs similarity index 100% rename from protocol-rs/src/consumer.rs rename to crates/protocol/src/consumer.rs diff --git a/protocol-rs/src/flow.rs b/crates/protocol/src/flow.rs similarity index 100% rename from protocol-rs/src/flow.rs rename to crates/protocol/src/flow.rs diff --git a/protocol-rs/src/gogoproto.rs b/crates/protocol/src/gogoproto.rs similarity index 100% rename from protocol-rs/src/gogoproto.rs rename to crates/protocol/src/gogoproto.rs diff --git a/protocol-rs/src/google.protobuf.rs b/crates/protocol/src/google.protobuf.rs similarity index 100% rename from protocol-rs/src/google.protobuf.rs rename to crates/protocol/src/google.protobuf.rs diff --git a/protocol-rs/src/lib.rs b/crates/protocol/src/lib.rs similarity index 100% rename from protocol-rs/src/lib.rs rename to crates/protocol/src/lib.rs diff --git a/protocol-rs/src/protocol.rs b/crates/protocol/src/protocol.rs similarity index 100% rename from protocol-rs/src/protocol.rs rename to crates/protocol/src/protocol.rs diff --git a/protocol-rs/src/read.rs b/crates/protocol/src/read.rs similarity index 100% rename from protocol-rs/src/read.rs rename to crates/protocol/src/read.rs diff --git a/protocol-rs/src/recoverylog.rs b/crates/protocol/src/recoverylog.rs similarity index 100% rename from protocol-rs/src/recoverylog.rs rename to crates/protocol/src/recoverylog.rs diff --git a/protocol-rs/src/snapshots/estuary_protocol__test__serde_round_trip_of_collection_spec.snap b/crates/protocol/src/snapshots/protocol__test__serde_round_trip_of_collection_spec.snap similarity index 100% rename from protocol-rs/src/snapshots/estuary_protocol__test__serde_round_trip_of_collection_spec.snap rename to crates/protocol/src/snapshots/protocol__test__serde_round_trip_of_collection_spec.snap diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml new file mode 100644 index 0000000000..c5d0428a8a --- /dev/null +++ b/crates/runtime/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "runtime" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +labels = { path = "../labels", version = "0.0.0" } +protocol = { path = "../protocol", version = "0.0.0" } + +chrono = { version = "*", features = ["serde"] } +futures = "*" +log = "*" +prost-types = "*" +rusqlite = { version = "*", features = ["bundled", "collation", "column_decltype", "functions", "serde_json", "url"] } +serde = { version = "*", features = ["derive"] } +serde_json = { version = "*"} +tempfile = "*" +thiserror = "*" +tokio = { version = "0.2", features = ["full"] } +tonic = { version = "*" } diff --git a/src/runtime/cluster.rs b/crates/runtime/src/cluster.rs similarity index 88% rename from src/runtime/cluster.rs rename to crates/runtime/src/cluster.rs index 507cffc5e7..bc04008f89 100644 --- a/src/runtime/cluster.rs +++ b/crates/runtime/src/cluster.rs @@ -1,8 +1,6 @@ -use estuary_protocol::consumer::{self, shard_client::ShardClient}; -use estuary_protocol::flow::{ - self, ingester_client::IngesterClient, testing_client::TestingClient, -}; -use estuary_protocol::protocol::{self, journal_client::JournalClient}; +use protocol::consumer::{self, shard_client::ShardClient}; +use protocol::flow::{self, ingester_client::IngesterClient, testing_client::TestingClient}; +use protocol::protocol::{self as broker, journal_client::JournalClient}; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -91,11 +89,11 @@ impl Cluster { pub async fn list_journals( &self, - selector: Option, - ) -> Result { + selector: Option, + ) -> Result { let mut client = JournalClient::connect(self.broker_address.clone()).await?; - let request = tonic::Request::new(protocol::ListRequest { selector }); + let request = tonic::Request::new(broker::ListRequest { selector }); let response = client.list(request).await?; let response = response.into_inner(); @@ -111,8 +109,8 @@ impl Cluster { pub async fn read( &self, - request: protocol::ReadRequest, - ) -> Result, Error> { + request: broker::ReadRequest, + ) -> Result, Error> { let mut client = JournalClient::connect(self.broker_address.clone()).await?; let response = client.read(request).await?; @@ -121,7 +119,7 @@ impl Cluster { pub async fn list_shards( &self, - selector: Option, + selector: Option, ) -> Result { let mut client = ShardClient::connect(self.consumer_address.clone()).await?; @@ -163,8 +161,8 @@ impl Cluster { pub async fn apply_journals( &self, - req: protocol::ApplyRequest, - ) -> Result { + req: broker::ApplyRequest, + ) -> Result { let mut client = JournalClient::connect(self.broker_address.clone()).await?; let request = tonic::Request::new(req); diff --git a/src/runtime/mod.rs b/crates/runtime/src/lib.rs similarity index 100% rename from src/runtime/mod.rs rename to crates/runtime/src/lib.rs diff --git a/src/runtime/local.rs b/crates/runtime/src/local.rs similarity index 93% rename from src/runtime/local.rs rename to crates/runtime/src/local.rs index 147d9e9be1..fc075cc8e1 100644 --- a/src/runtime/local.rs +++ b/crates/runtime/src/local.rs @@ -184,22 +184,15 @@ impl Local { pub async fn stop(mut self) -> Result<(), Error> { log::info!("stopping local runtime"); - self.consumer.kill()?; - let status = self.consumer.await?; - log::info!("consumer exited: {}", status); - - self.ingester.kill()?; - let status = self.ingester.await?; - log::info!("ingester exited: {}", status); - - self.gazette.kill()?; - let status = self.gazette.await?; - log::info!("gazette exited: {}", status); - - self.etcd.kill()?; - let status = self.etcd.await?; - log::info!("etcd exited: {}", status); - + for (name, child) in &mut [ + ("consumer", &mut self.consumer), + ("ingester", &mut self.ingester), + ("gazette", &mut self.gazette), + ("etcd", &mut self.etcd), + ] { + let _ = child.kill(); + log::info!("{} exited: {}", name, child.await?); + } Ok(()) } } @@ -250,7 +243,7 @@ impl Log { name: &str, child: &mut tokio::process::Child, ) -> impl Stream> { - let name = format!("estuary::{}<{}>", name, child.id()); + let name = format!("runtime::{}<{}>", name, child.id()); BufReader::new(child.stderr.take().unwrap()) .lines() diff --git a/src/runtime/specs.rs b/crates/runtime/src/specs.rs similarity index 88% rename from src/runtime/specs.rs rename to crates/runtime/src/specs.rs index a597df64e2..4528127108 100644 --- a/src/runtime/specs.rs +++ b/crates/runtime/src/specs.rs @@ -1,5 +1,6 @@ -use crate::{catalog, label_set, labels::keys as label_keys, labels::values as label_values}; -use estuary_protocol::{consumer, protocol}; +use labels::{keys as label_keys, label_set, values as label_values}; +use protocol::{consumer, protocol as broker}; +use rusqlite::Connection as DB; use std::collections::BTreeMap; use std::fmt::{Display, Write}; @@ -29,7 +30,7 @@ fn derivation_shard_id( } impl DerivationSet { - pub fn update_from_catalog(&mut self, db: &catalog::DB) -> catalog::Result<()> { + pub fn update_from_catalog(&mut self, db: &DB) -> rusqlite::Result<()> { let mut stmt = db.prepare("SELECT collection_name FROM collections_json WHERE is_derivation")?; let mut rows = stmt.query(rusqlite::NO_PARAMS)?; @@ -40,7 +41,7 @@ impl DerivationSet { Ok(()) } - pub fn build_recovery_log_apply_request(&self) -> protocol::ApplyRequest { + pub fn build_recovery_log_apply_request(&self) -> broker::ApplyRequest { let changes = self .0 .iter() @@ -50,9 +51,9 @@ impl DerivationSet { "content-type" => "application/x-gazette-recoverylog", }); - let fragment = Some(protocol::journal_spec::Fragment { + let fragment = Some(broker::journal_spec::Fragment { length: 1 << 28, // 256MB. - compression_codec: (protocol::CompressionCodec::None as i32), + compression_codec: (broker::CompressionCodec::None as i32), stores: vec!["file:///".to_owned()], refresh_interval: Some(std::time::Duration::from_secs(5 * 60).into()), retention: None, @@ -60,8 +61,8 @@ impl DerivationSet { path_postfix_template: String::new(), }); - protocol::apply_request::Change { - upsert: Some(protocol::JournalSpec { + broker::apply_request::Change { + upsert: Some(broker::JournalSpec { name: format!( "recovery/{}", derivation_shard_id( @@ -82,7 +83,7 @@ impl DerivationSet { }) .collect::>(); - protocol::ApplyRequest { changes } + broker::ApplyRequest { changes } } pub fn build_shard_apply_request(&self, catalog_url: &str) -> consumer::ApplyRequest { diff --git a/crates/serve/Cargo.toml b/crates/serve/Cargo.toml new file mode 100644 index 0000000000..eab9611df5 --- /dev/null +++ b/crates/serve/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "serve" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +http = "*" +http-body = "*" +hyper = "*" +hyperlocal = "*" +log = "*" +tower = "*" +# At this time, hyper & hyperlocal depend on tokio 0.2 and not 0.3. +# This crate fails to build with tokio 0.3. +tokio = { version = "0.2", features = ["full"] } + +[dev-dependencies] +# TODO(johnny): Remove me. It's used for a couple of unit tests only +# but brings in a lot of dependencies. Replace with a hyper server? +warp = "*" +tempfile = "*" \ No newline at end of file diff --git a/src/serve/mod.rs b/crates/serve/src/lib.rs similarity index 100% rename from src/serve/mod.rs rename to crates/serve/src/lib.rs diff --git a/src/serve/unix.rs b/crates/serve/src/unix.rs similarity index 96% rename from src/serve/unix.rs rename to crates/serve/src/unix.rs index dd8614cdfe..651e695db1 100644 --- a/src/serve/unix.rs +++ b/crates/serve/src/unix.rs @@ -24,7 +24,7 @@ where B::Data: Send, B::Error: std::error::Error + Send + Sync + 'static, { - let make_svc = make_service_fn(move |stream: &tokio::net::UnixStream| { + let make_svc = make_service_fn(move |stream| { debug!("socket connected {:?}", stream); let svc = svc.clone(); @@ -61,7 +61,6 @@ mod test { #[tokio::test] async fn test_with_simple_server() { - let _ = pretty_env_logger::init(); let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("test-sock"); diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml new file mode 100644 index 0000000000..399d4ec8a6 --- /dev/null +++ b/crates/testing/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "testing" +version = "0.0.0" +authors = ["Estuary Technologies, Inc"] +edition = "2018" + +[dependencies] +catalog = { path = "../catalog", version = "0.0.0" } +doc = { path = "../doc", version = "0.0.0" } +json = { path = "../json", version = "0.0.0" } +derive = { path = "../derive", version = "0.0.0" } +runtime = { path = "../runtime", version = "0.0.0" } +protocol = { path = "../protocol", version = "0.0.0" } + +futures = "*" +itertools = "*" +log = "*" +percent-encoding = "*" +prost = "*" +rusqlite = { version = "*", features = ["bundled", "collation", "column_decltype", "functions", "serde_json", "url"] } +serde = { version = "*", features = ["derive"] } +serde_json = { version = "*"} +thiserror = "*" +tonic = { version = "*" } +url = "*" \ No newline at end of file diff --git a/src/testing/driver.rs b/crates/testing/src/driver.rs similarity index 99% rename from src/testing/driver.rs rename to crates/testing/src/driver.rs index 579b1a0e84..d97eb9528d 100644 --- a/src/testing/driver.rs +++ b/crates/testing/src/driver.rs @@ -1,12 +1,12 @@ -use crate::catalog::specs::{TestStep, TestStepIngest, TestStepVerify}; -use estuary_protocol::consumer; -use estuary_protocol::flow; -use estuary_protocol::protocol::header; +use catalog::specs::{TestStep, TestStepIngest, TestStepVerify}; use itertools::{ EitherOrBoth::{Both, Left, Right}, Itertools, }; use prost::Message; +use protocol::consumer; +use protocol::flow; +use protocol::protocol::header; use serde::Deserialize; use std::collections::{BTreeMap, BTreeSet}; diff --git a/src/testing/mod.rs b/crates/testing/src/lib.rs similarity index 97% rename from src/testing/mod.rs rename to crates/testing/src/lib.rs index 984731edcc..773d136e81 100644 --- a/src/testing/mod.rs +++ b/crates/testing/src/lib.rs @@ -1,17 +1,13 @@ -use crate::catalog::{ - self, - specs::{TestStep, TestStepVerify}, -}; -use crate::derive; -use crate::doc::{reduce, Diff, Pointer, SchemaIndex}; -use crate::runtime::{self, cluster}; -use estuary_json::Location; -use estuary_protocol::consumer; -use estuary_protocol::flow::{self, ingest_request, testing_client::TestingClient}; -use estuary_protocol::protocol::{Label, LabelSelector, LabelSet, ReadRequest}; +use catalog::specs::{TestStep, TestStepVerify}; +use doc::{reduce, Diff, Pointer, SchemaIndex}; use futures::stream::futures_unordered::FuturesUnordered; use futures::{StreamExt, TryStreamExt}; use itertools::Itertools; +use json::Location; +use protocol::consumer; +use protocol::flow::{self, ingest_request, testing_client::TestingClient}; +use protocol::protocol::{Label, LabelSelector, LabelSet, ReadRequest}; +use runtime::cluster; use serde_json::Value; use std::collections::{BTreeMap, BTreeSet}; diff --git a/flow.schema.json b/flow.schema.json deleted file mode 100644 index 721a861a31..0000000000 --- a/flow.schema.json +++ /dev/null @@ -1,884 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Catalog", - "description": "Each catalog source defines a portion of a Flow Catalog, by defining collections, derivations, tests, and materializations of the Catalog. Catalog sources may reference and import other sources, in order to collections and other entities that source defines.", - "type": "object", - "properties": { - "$schema": { - "title": "JSON-Schema against which the Catalog is validated.", - "default": null, - "type": [ - "string", - "null" - ] - }, - "collections": { - "title": "Collections defined by this Catalog.", - "default": [], - "examples": [ - [ - { - "derivation": null, - "key": [ - "/json/ptr" - ], - "name": "a/collection", - "projections": {}, - "schema": "../path/to/local.yaml" - } - ] - ], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "import": { - "title": "Import other Flow catalog sources.", - "description": "By importing another Flow catalog source, the collections, schemas, and derivations it defines become usable within this Catalog source. Each import is an absolute URI, or a URI which is relative to this source location.", - "default": [], - "examples": [ - [ - "../path/to/local.yaml", - "https://example/resource" - ] - ], - "type": "array", - "items": { - "$ref": "#/definitions/RelativeUrl" - } - }, - "materializations": { - "title": "Materializations defined by this catalog.", - "description": "Materializations project a view of the current state into an external system like a database or key/value store. These states will be kept up to date automatically as documents are processed in the collection. The keys used here are arbitrary identifiers that will be used to uniquely identify each materialization, and the values are any valid Materialization object.", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/Materialization" - } - }, - "nodeDependencies": { - "title": "NPM package dependencies of the Catalog.", - "description": "Dependencies are included when building the catalog's build NodeJS package, as {\"package-name\": \"version\"}. I.e. {\"moment\": \"^2.24\"}.\n\nVersion strings can take any form understood by NPM. See https://docs.npmjs.com/files/package.json#dependencies", - "default": { - "a-npm-package": "^1.2.3" - }, - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "tests": { - "title": "Tests defined by this Catalog.", - "description": "Tests are keyed by their test name, and are defined in terms of a series of sequential test steps.", - "default": { - "Test that fob quips ipsum": [] - }, - "examples": [ - { - "Test that fob quips ipsum": [ - { - "ingest": { - "collection": "a/collection", - "documents": [ - { - "example": "document" - }, - { - "another": "document" - } - ] - } - }, - { - "verify": { - "collection": "a/collection", - "documents": [ - { - "expected": "document" - } - ], - "partitions": null - } - } - ] - } - ], - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "$ref": "#/definitions/TestStep" - } - } - } - }, - "additionalProperties": false, - "definitions": { - "Collection": { - "description": "Collection describes a set of related documents, where each adheres to a common schema and grouping key. Collections are append-only: once a document is added to a collection, it is never removed. However, it may be replaced or updated (either in whole, or in part) by a future document sharing its key. Each new document of a given key is \"reduced\" into existing documents of the key. By default, this reduction is achieved by completely replacing the previous document, but much richer reduction behaviors can be specified through the use of annotated reduction strategies of the collection schema.", - "examples": [ - { - "derivation": null, - "key": [ - "/json/ptr" - ], - "name": "a/collection", - "projections": {}, - "schema": "../path/to/local.yaml" - } - ], - "type": "object", - "required": [ - "key", - "name", - "schema" - ], - "properties": { - "derivation": { - "title": "Derivation which builds this collection from others.", - "anyOf": [ - { - "$ref": "#/definitions/Derivation" - }, - { - "type": "null" - } - ] - }, - "key": { - "title": "Composite key of this collection.", - "allOf": [ - { - "$ref": "#/definitions/CompositeKey" - } - ] - }, - "name": { - "title": "Canonical name of this collection.", - "allOf": [ - { - "$ref": "#/definitions/CollectionName" - } - ] - }, - "projections": { - "title": "Projections and logical partitions of this collection.", - "default": { - "a_field": "/json/ptr", - "a_partition": { - "location": "/json/ptr", - "partition": true - } - }, - "allOf": [ - { - "$ref": "#/definitions/Projections" - } - ] - }, - "schema": { - "title": "Schema against which collection documents are validated and reduced.", - "examples": [ - "../path/to/schema#/$defs/subPath" - ], - "allOf": [ - { - "$ref": "#/definitions/Schema" - } - ] - } - }, - "additionalProperties": false - }, - "CollectionName": { - "description": "Names consist of Unicode letters, numbers, and symbols: - _ . /\n\nSpaces and other special characters are disallowed.", - "examples": [ - "a/collection" - ], - "type": "string", - "pattern": "^[^ \t\n\\!@#$%^&*()+=\\<\\>?;:'\"\\[\\]\\|~`]+$" - }, - "CompositeKey": { - "description": "Ordered JSON-Pointers which define how a composite key may be extracted from a collection document.", - "examples": [ - [ - "/json/ptr" - ] - ], - "type": "array", - "items": { - "$ref": "#/definitions/JsonPointer" - } - }, - "Derivation": { - "description": "A derivation specifies how a collection is derived from other collections. A collection without a derivation is a \"captured\" collection, into which documents are directly ingested.", - "examples": [ - { - "bootstrap": [], - "register": { - "initial": null, - "schema": true - }, - "transform": { - "nameOfTransform": { - "publish": { - "nodeJS": "return doPublish(source, register);" - }, - "readDelay": null, - "shuffle": null, - "source": { - "name": "a/collection", - "partitions": null, - "schema": null - }, - "update": { - "nodeJS": "return doUpdate(source);" - } - } - } - } - ], - "type": "object", - "properties": { - "bootstrap": { - "title": "Bootstrap lambdas of this derivation.", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Lambda" - } - }, - "register": { - "title": "Register configuration of this derivation.", - "default": { - "initial": null, - "schema": true - }, - "allOf": [ - { - "$ref": "#/definitions/Register" - } - ] - }, - "transform": { - "title": "Transforms which make up this derivation.", - "default": { - "nameOfTransform": { - "source": { - "name": "a/source/collection" - } - } - }, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/Transform" - } - } - }, - "additionalProperties": false - }, - "FullProjection": { - "description": "A full projection description which may specify a logical partition.", - "examples": [ - { - "location": "/json/ptr", - "partition": true - } - ], - "type": "object", - "required": [ - "location" - ], - "properties": { - "location": { - "title": "Location of this projection.", - "allOf": [ - { - "$ref": "#/definitions/JsonPointer" - } - ] - }, - "partition": { - "title": "Is this projection a logical partition?", - "default": false, - "type": "boolean" - } - }, - "additionalProperties": false - }, - "JsonPointer": { - "description": "JSON Pointer which identifies a location in a document.", - "examples": [ - "/json/ptr" - ], - "type": "string", - "pattern": "^/.+" - }, - "Lambda": { - "description": "Lambdas are user functions which are invoked by the Flow runtime to process and transform source collection documents into derived collections. Flow supports multiple lambda run-times, with a current focus on TypeScript and remote HTTP APIs.\n\nTypeScript lambdas are invoked within on-demand run-times, which are automatically started and scaled by Flow's task distribution in order to best co-locate data and processing, as well as to manage fail-over.\n\nRemote lambdas may be called from many Flow tasks, and are up to the API provider to provision and scale.\n\n(Note that Sqlite lambdas are not implemented yet).\n\nLambdas are invoked from a few contexts:\n\n\"Bootstrap\" lambdas are run once, at the start of a Flow task assignment, and before it processes any source documents. They provide an opportunity to initialize temporary state or ephemeral caches.\n\n\"Update\" lambdas take a source document and transform it into one or more register updates, which are then reduced into the associated register by the runtime. For example these register updates might update counters, or update the state of a \"join\" window.\n\n\"Publish\" lambdas take a source document, a current register and (if there is also an \"update\" lambda) a previous register, and transform them into one or more documents to be published into a derived collection.", - "examples": [ - { - "nodeJS": "return doPublish(source, register);" - }, - { - "nodeJS": "return doUpdate(source);" - }, - { - "nodeJS": "console.error('example of a bootstrap');" - }, - { - "remote": "http://example/api" - } - ], - "anyOf": [ - { - "type": "object", - "required": [ - "nodeJS" - ], - "properties": { - "nodeJS": { - "type": "string" - } - } - }, - { - "type": "object", - "required": [ - "sqlite" - ], - "properties": { - "sqlite": { - "type": "string" - } - } - }, - { - "type": "object", - "required": [ - "sqliteFile" - ], - "properties": { - "sqliteFile": { - "$ref": "#/definitions/RelativeUrl" - } - } - }, - { - "type": "object", - "required": [ - "remote" - ], - "properties": { - "remote": { - "type": "string" - } - } - } - ] - }, - "Materialization": { - "description": "A materialization represents the desire to maintain a continuously updated state of the documents in a collection.", - "type": "object", - "anyOf": [ - { - "type": "object", - "required": [ - "postgres" - ], - "properties": { - "postgres": { - "$ref": "#/definitions/SqlTargetConnection" - } - } - }, - { - "type": "object", - "required": [ - "sqlite" - ], - "properties": { - "sqlite": { - "$ref": "#/definitions/SqlTargetConnection" - } - } - } - ], - "required": [ - "collection" - ], - "properties": { - "collection": { - "description": "The name of the collection to materialize. This must exactly match the name of a collection that exists in either in this catalog, or in another catalog imported by this one.", - "allOf": [ - { - "$ref": "#/definitions/CollectionName" - } - ] - } - } - }, - "PartitionSelector": { - "description": "Partition selectors identify a desired subset of the available logical partitions of a collection.", - "examples": [ - { - "exclude": { - "other_partition": [ - 32, - 64 - ] - }, - "include": { - "a_partition": [ - "A", - "B" - ] - } - } - ], - "type": "object", - "properties": { - "exclude": { - "description": "Partition field names and values which are excluded from the source collection. Any documents matching *any one* of the partition values will be excluded.", - "default": {}, - "type": "object", - "additionalProperties": { - "type": "array", - "items": true - } - }, - "include": { - "description": "Partition field names and corresponding values which must be matched from the Source collection. Only documents having one of the specified values across all specified partition names will be matched. For example, source: [App, Web] region: [APAC] would mean only documents of 'App' or 'Web' source and also occurring in the 'APAC' region will be processed.", - "default": {}, - "type": "object", - "additionalProperties": { - "type": "array", - "items": true - } - } - }, - "additionalProperties": false - }, - "Projection": { - "description": "A projection representation that allows projections to be specified either as a simple JSON Pointer, or as an object with separate properties for `loction` and `partition`.", - "anyOf": [ - { - "$ref": "#/definitions/JsonPointer" - }, - { - "$ref": "#/definitions/FullProjection" - } - ] - }, - "Projections": { - "description": "Projections are named locations within a collection document which may be used for logical partitioning or directly exposed to databases into which collections are materialized.", - "examples": [ - { - "a_field": "/json/ptr", - "a_partition": { - "location": "/json/ptr", - "partition": true - } - } - ], - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/Projection" - } - }, - "Register": { - "description": "Registers are the internal states of a derivation, which can be read and updated by all of its transformations. They're an important building block for joins, aggregations, and other complex stateful workflows.\n\nRegisters are implemented using JSON-Schemas, often ones with reduction annotations. When reading source documents, every distinct shuffle key by which the source collection is read is mapped to a corresponding register value (or, if no shuffle key is defined, the source collection's key is used instead).\n\nThen, an \"update\" lambda of the transformation produces updates which are reduced into the register, and a \"publish\" lambda reads the current (and previous, if updated) register value.", - "type": "object", - "required": [ - "schema" - ], - "properties": { - "initial": { - "title": "Initial value of a keyed register which has never been updated.", - "description": "If not specified, the default is \"null\".", - "default": null - }, - "schema": { - "title": "Schema which validates and reduces register documents.", - "allOf": [ - { - "$ref": "#/definitions/Schema" - } - ] - } - }, - "additionalProperties": false - }, - "RelativeUrl": { - "description": "A URL identifying a resource, which may be a relative local path with respect to the current resource (i.e, ../path/to/flow.yaml), or may be an external absolute URL (i.e., http://example/flow.yaml).", - "examples": [ - "../path/to/local.yaml", - "https://example/resource" - ], - "type": "string" - }, - "Schema": { - "description": "A schema is a draft 2019-09 JSON Schema which validates Flow documents. Schemas also provide annotations at document locations, such as reduction strategies for combining one document into another.\n\nSchemas may be defined inline to the catalog, or given as a relative or absolute URI. URIs may optionally include a JSON fragment pointer that locates a specific sub-schema therein.\n\nI.e, \"schemas/marketing.yaml#/$defs/campaign\" would reference the schema at location {\"$defs\": {\"campaign\": ...}} within ./schemas/marketing.yaml.", - "examples": [ - "http://example/schema#/$defs/subPath", - "../path/to/schema#/$defs/subPath", - { - "properties": { - "bar": { - "const": 42 - }, - "foo": { - "type": "integer" - } - }, - "type": "object" - }, - { - "properties": { - "foo_count": { - "reduce": { - "strategy": "sum" - }, - "type": "integer" - } - }, - "reduce": { - "strategy": "merge" - }, - "type": "object" - } - ], - "anyOf": [ - { - "$ref": "#/definitions/RelativeUrl" - }, - { - "type": "object", - "additionalProperties": true - }, - { - "type": "boolean" - } - ] - }, - "Source": { - "examples": [ - { - "name": "a/collection", - "partitions": null, - "schema": null - } - ], - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "title": "Name of the collection to be read.", - "allOf": [ - { - "$ref": "#/definitions/CollectionName" - } - ] - }, - "partitions": { - "title": "Selector over partition of the source collection to read.", - "default": { - "exclude": { - "other_partition": [ - 32, - 64 - ] - }, - "include": { - "a_partition": [ - "A", - "B" - ] - } - }, - "anyOf": [ - { - "$ref": "#/definitions/PartitionSelector" - }, - { - "type": "null" - } - ] - }, - "schema": { - "title": "Optional JSON-Schema to validate against the source collection.", - "description": "All data in the source collection is already validated against the schema of that collection, so providing a source schema is only used for _additional_ validation beyond that.\n\nThis is useful in building \"Extract Load Transform\" patterns, where a collection is captured with minimal schema applied (perhaps because it comes from an uncontrolled third party), and is then progressively verified as collections are derived. If None, the principal schema of the collection is used instead.", - "default": "../path/to/schema#/$defs/subPath", - "anyOf": [ - { - "$ref": "#/definitions/Schema" - }, - { - "type": "null" - } - ] - } - }, - "additionalProperties": false - }, - "SqlTargetConnection": { - "type": "object", - "required": [ - "table", - "uri" - ], - "properties": { - "table": { - "description": "The name of the table to materialize into.", - "type": "string" - }, - "uri": { - "description": "The connection URI for the target database, e.g. `postgresql://user:password@my-postgres.test:5432/my_database`", - "type": "string" - } - } - }, - "TestStep": { - "examples": [ - { - "ingest": { - "collection": "a/collection", - "documents": [ - { - "example": "document" - }, - { - "another": "document" - } - ] - } - }, - { - "verify": { - "collection": "a/collection", - "documents": [ - { - "expected": "document" - } - ], - "partitions": null - } - } - ], - "anyOf": [ - { - "description": "Ingest document fixtures into a collection.", - "type": "object", - "required": [ - "ingest" - ], - "properties": { - "ingest": { - "$ref": "#/definitions/TestStepIngest" - } - } - }, - { - "description": "Verify the contents of a collection match a set of document fixtures.", - "type": "object", - "required": [ - "verify" - ], - "properties": { - "verify": { - "$ref": "#/definitions/TestStepVerify" - } - } - } - ] - }, - "TestStepIngest": { - "description": "An ingestion test step ingests document fixtures into the named collection.", - "examples": [ - { - "collection": "a/collection", - "documents": [ - { - "example": "document" - }, - { - "another": "document" - } - ] - } - ], - "type": "object", - "required": [ - "collection", - "documents" - ], - "properties": { - "collection": { - "title": "Name of the collection into which the test will ingest.", - "allOf": [ - { - "$ref": "#/definitions/CollectionName" - } - ] - }, - "documents": { - "title": "Documents to ingest.", - "description": "Each document must conform to the collection's schema.", - "type": "array", - "items": true - } - } - }, - "TestStepVerify": { - "description": "A verification test step verifies that the contents of the named collection match the expected fixtures, after fully processing all preceding ingestion test steps.", - "examples": [ - { - "collection": "a/collection", - "documents": [ - { - "expected": "document" - } - ], - "partitions": null - } - ], - "type": "object", - "required": [ - "collection", - "documents" - ], - "properties": { - "collection": { - "title": "Collection into which the test will ingest.", - "allOf": [ - { - "$ref": "#/definitions/CollectionName" - } - ] - }, - "documents": { - "title": "Documents to verify.", - "description": "Each document may contain only a portion of the matched document's properties, and any properties present in the actual document but not in this document fixture are ignored. All other values must match or the test will fail.", - "type": "array", - "items": true - }, - "partitions": { - "title": "Selector over partitions to verify.", - "default": { - "exclude": { - "other_partition": [ - 32, - 64 - ] - }, - "include": { - "a_partition": [ - "A", - "B" - ] - } - }, - "anyOf": [ - { - "$ref": "#/definitions/PartitionSelector" - }, - { - "type": "null" - } - ] - } - } - }, - "Transform": { - "description": "A Transform reads and shuffles documents of a source collection, and processes each document through either one or both of a register \"update\" lambda and a derived document \"publish\" lambda.", - "examples": [ - { - "publish": { - "nodeJS": "return doPublish(source, register);" - }, - "readDelay": null, - "shuffle": null, - "source": { - "name": "a/collection", - "partitions": null, - "schema": null - }, - "update": { - "nodeJS": "return doUpdate(source);" - } - } - ], - "type": "object", - "required": [ - "source" - ], - "properties": { - "publish": { - "title": "Publish lambda that produces documents to publish into the collection.", - "default": { - "nodeJS": "return doPublish(source, register);" - }, - "anyOf": [ - { - "$ref": "#/definitions/Lambda" - }, - { - "type": "null" - } - ] - }, - "readDelay": { - "title": "Delay applied to documents read by this transform.", - "description": "Delays are applied as an adjustment to the UUID clock encoded within each document, which is then used to impose a relative ordering of all documents read by this derivation. This means that read delays are applied in a consistent way, even when back-filling over historical documents. When caught up and tailing the source collection, delays also \"gate\" documents such that they aren't processed until the current wall-time reflects the delay.", - "default": null, - "type": [ - "string", - "null" - ], - "pattern": "^\\d+(s|m|h)$" - }, - "shuffle": { - "title": "Shuffle key by which source documents are mapped to registers.", - "description": "If empty, the key of the source collection is used.", - "default": [ - "/json/ptr" - ], - "anyOf": [ - { - "$ref": "#/definitions/CompositeKey" - }, - { - "type": "null" - } - ] - }, - "source": { - "title": "Source collection read by this transform.", - "allOf": [ - { - "$ref": "#/definitions/Source" - } - ] - }, - "update": { - "title": "Update lambda that produces register updates from source documents.", - "default": { - "nodeJS": "return doUpdate(source);" - }, - "anyOf": [ - { - "$ref": "#/definitions/Lambda" - }, - { - "type": "null" - } - ] - } - }, - "additionalProperties": false - } - } -} \ No newline at end of file diff --git a/cmd/flow-consumer/main.go b/go/flow-consumer/main.go similarity index 100% rename from cmd/flow-consumer/main.go rename to go/flow-consumer/main.go diff --git a/cmd/flow-ingester/main.go b/go/flow-ingester/main.go similarity index 100% rename from cmd/flow-ingester/main.go rename to go/flow-ingester/main.go diff --git a/json/Cargo.lock b/json/Cargo.lock deleted file mode 100644 index 814a2b5c31..0000000000 --- a/json/Cargo.lock +++ /dev/null @@ -1,249 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -[[package]] -name = "aho-corasick" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" -dependencies = [ - "memchr", -] - -[[package]] -name = "byteorder" -version = "1.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" - -[[package]] -name = "either" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" - -[[package]] -name = "estuary_json" -version = "0.1.0" -dependencies = [ - "fxhash", - "glob", - "itertools", - "percent-encoding", - "regex", - "serde", - "serde_json", - "thiserror", - "tinyvec", - "url", -] - -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] -name = "glob" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" - -[[package]] -name = "idna" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "matches" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" - -[[package]] -name = "memchr" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" - -[[package]] -name = "percent-encoding" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" - -[[package]] -name = "proc-macro2" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "quote" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "regex" -version = "1.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", - "thread_local", -] - -[[package]] -name = "regex-syntax" -version = "0.6.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" - -[[package]] -name = "ryu" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" - -[[package]] -name = "serde" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" - -[[package]] -name = "serde_json" -version = "1.0.51" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "smallvec" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" - -[[package]] -name = "syn" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "410a7488c0a728c7ceb4ad59b9567eb4053d02e8cc7f5c0e0eeeb39518369213" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - -[[package]] -name = "thiserror" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b3d3d2ff68104100ab257bb6bb0cb26c901abe4bd4ba15961f3bf867924012" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca972988113b7715266f91250ddb98070d033c62a011fa0fcc57434a649310dd" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thread_local" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "tinyvec" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" - -[[package]] -name = "unicode-bidi" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" -dependencies = [ - "matches", -] - -[[package]] -name = "unicode-normalization" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" -dependencies = [ - "smallvec", -] - -[[package]] -name = "unicode-xid" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" - -[[package]] -name = "url" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" -dependencies = [ - "idna", - "matches", - "percent-encoding", -] diff --git a/json/Cargo.toml b/json/Cargo.toml deleted file mode 100644 index 2f5be88db2..0000000000 --- a/json/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "estuary_json" -version = "0.1.0" -authors = ["Johnny Graettinger "] -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -url = "2.1.1" -percent-encoding = "2.1.0" -serde = "1.0" -serde_json = "1.0" -regex = "1.3.1" -fxhash = "0.2.1" -tinyvec = {version = "0.3.3", features = ["alloc"]} -thiserror = "1.0.11" -itertools = "0.9" -serde_yaml = "0.8" - -[dev-dependencies] -glob = "0.3.0" - -[lib] -path = "lib.rs" diff --git a/protocol-rs/Cargo.toml b/protocol-rs/Cargo.toml deleted file mode 100644 index deb202ec9e..0000000000 --- a/protocol-rs/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "estuary_protocol" -version = "0.1.0" -authors = ["Johnny Graettinger "] -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -serde = "1.0" -tonic = "0.3" -prost = "0.6" -prost-types = "0.6" -humantime-serde = "1.0" - -[dev-dependencies] -insta = "^0.16" -serde_json = "1.0" - -[build-dependencies] -tonic-build = "0.3" diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index 8bc7a9a407..0000000000 --- a/src/lib.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Increase limit for use by select! in derive_api.rs. -#![recursion_limit = "256"] - -pub static FLOW_VERSION: &'static str = env!("CARGO_PKG_VERSION"); - -pub mod catalog; -pub mod derive; -pub mod doc; -pub mod labels; -pub mod materialization; -pub mod runtime; -pub mod serve; -pub mod testing; diff --git a/src/log.rs b/src/log.rs deleted file mode 100644 index 6bdd72dd89..0000000000 --- a/src/log.rs +++ /dev/null @@ -1,26 +0,0 @@ - -/* -use slog::{o, Drain}; -use std::sync::Once; - -pub fn log() -> &'static slog::Logger { - _LOG_INIT.call_once(|| { - /* - // Fancy version, that processes in another thread but can lose messages - // with quick-running tests. - let decorator = slog_term::TermDecorator::new().stderr().build(); - let drain = slog_term::FullFormat::new(decorator).build().fuse(); - let drain = slog_async::Async::new(drain).build().fuse(); - */ - let decorator = slog_term::PlainSyncDecorator::new(std::io::stderr()); - let drain = slog_term::FullFormat::new(decorator).build().fuse(); - unsafe { - _LOG = Some(slog::Logger::root(drain, o!("app" => "derive"))); - } - }); - unsafe { _LOG.as_ref().unwrap() } -} - -static _LOG_INIT: Once = Once::new(); -static mut _LOG: Option = None; -*/