From 40f2b422dacdb2547bf3cb871efc3bedb021b44a Mon Sep 17 00:00:00 2001 From: Jon Wihl Date: Wed, 13 Mar 2024 12:35:39 -0400 Subject: [PATCH 01/96] Added webhook variants --- source-http-ingest/VARIANTS | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 source-http-ingest/VARIANTS diff --git a/source-http-ingest/VARIANTS b/source-http-ingest/VARIANTS new file mode 100644 index 0000000000..6a7f471c77 --- /dev/null +++ b/source-http-ingest/VARIANTS @@ -0,0 +1,2 @@ +source-datadog-ingest +source-intercom-ingest \ No newline at end of file From 8e6cf0c4126ee7ea94df4a35198f64315104907e Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Tue, 12 Mar 2024 16:42:21 -0500 Subject: [PATCH 02/96] source-mysql: Add a test repro of the enum decoding bug Observe how after replacing the checkpointed enum metadata with the old-style variant which places `""` at the end of the list the decoded values in the second replication snapshot are wrong. --- .../.snapshots/TestEnumDecodingFix-backfill | 13 +++++++++++ .../TestEnumDecodingFix-replication1 | 13 +++++++++++ .../TestEnumDecodingFix-replication2 | 13 +++++++++++ source-mysql/capture_test.go | 22 +++++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 source-mysql/.snapshots/TestEnumDecodingFix-backfill create mode 100644 source-mysql/.snapshots/TestEnumDecodingFix-replication1 create mode 100644 source-mysql/.snapshots/TestEnumDecodingFix-replication2 diff --git a/source-mysql/.snapshots/TestEnumDecodingFix-backfill b/source-mysql/.snapshots/TestEnumDecodingFix-backfill new file mode 100644 index 0000000000..eebff8de92 --- /dev/null +++ b/source-mysql/.snapshots/TestEnumDecodingFix-backfill @@ -0,0 +1,13 @@ +# ================================ +# Collection "acmeCo/test/test_enumdecodingfix_32314857": 5 Documents +# ================================ +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"EnumDecodingFix_32314857","cursor":"backfill:0"}},"category":"A","id":1} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"EnumDecodingFix_32314857","cursor":"backfill:1"}},"category":"B","id":2} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"EnumDecodingFix_32314857","cursor":"backfill:2"}},"category":"C","id":3} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"EnumDecodingFix_32314857","cursor":"backfill:3"}},"category":"D","id":4} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"EnumDecodingFix_32314857","cursor":"backfill:4"}},"category":"","id":5} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test%2FEnumDecodingFix_32314857":{"backfilled":5,"key_columns":["id"],"metadata":{"schema":{"columns":["id","category"],"types":{"category":{"enum":["","A","C","B","D"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} + diff --git a/source-mysql/.snapshots/TestEnumDecodingFix-replication1 b/source-mysql/.snapshots/TestEnumDecodingFix-replication1 new file mode 100644 index 0000000000..24e3c07222 --- /dev/null +++ b/source-mysql/.snapshots/TestEnumDecodingFix-replication1 @@ -0,0 +1,13 @@ +# ================================ +# Collection "acmeCo/test/test_enumdecodingfix_32314857": 5 Documents +# ================================ +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"A","id":6} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"B","id":7} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"C","id":8} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"D","id":9} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"","id":10} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test%2FEnumDecodingFix_32314857":{"backfilled":5,"key_columns":["id"],"metadata":{"schema":{"columns":["id","category"],"types":{"category":{"enum":["","A","C","B","D"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} + diff --git a/source-mysql/.snapshots/TestEnumDecodingFix-replication2 b/source-mysql/.snapshots/TestEnumDecodingFix-replication2 new file mode 100644 index 0000000000..da351a92c7 --- /dev/null +++ b/source-mysql/.snapshots/TestEnumDecodingFix-replication2 @@ -0,0 +1,13 @@ +# ================================ +# Collection "acmeCo/test/test_enumdecodingfix_32314857": 5 Documents +# ================================ +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"C","id":11} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"D","id":12} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"B","id":13} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"","id":14} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"A","id":15} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test%2FEnumDecodingFix_32314857":{"backfilled":5,"key_columns":["id"],"metadata":{"schema":{"columns":["id","category"],"types":{"category":{"enum":["A","C","B","D",""],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} + diff --git a/source-mysql/capture_test.go b/source-mysql/capture_test.go index 868c5e3c50..965b1c83d9 100644 --- a/source-mysql/capture_test.go +++ b/source-mysql/capture_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "regexp" + "strings" "testing" st "github.com/estuary/connectors/source-boilerplate/testing" @@ -180,3 +181,24 @@ func TestEnumPrimaryKey(t *testing.T) { }) tests.VerifiedCapture(ctx, t, cs) } + +func TestEnumDecodingFix(t *testing.T) { + var tb, ctx = mysqlTestBackend(t), context.Background() + var uniqueID = "32314857" + var tableName = tb.CreateTable(ctx, t, uniqueID, "(id INTEGER PRIMARY KEY, category ENUM('A', 'C', 'B', 'D'))") + var cs = tb.CaptureSpec(ctx, t, regexp.MustCompile(uniqueID)) + cs.Validator = &st.OrderedCaptureValidator{} + + // Insert various test values and then capture them via replication + tb.Insert(ctx, t, tableName, [][]interface{}{{1, "A"}, {2, "B"}, {3, "C"}, {4, "D"}, {5, "error"}}) + t.Run("backfill", func(t *testing.T) { tests.VerifiedCapture(ctx, t, cs) }) + tb.Insert(ctx, t, tableName, [][]interface{}{{6, "A"}, {7, "B"}, {8, "C"}, {9, "D"}, {10, "error"}}) + t.Run("replication1", func(t *testing.T) { tests.VerifiedCapture(ctx, t, cs) }) + + // Manually fiddle with the persisted checkpoint metadata used for enum decoding, to + // simulate the situation where an old capture with old metadata is used with the newer + // enum decoding logic. + cs.Checkpoint = json.RawMessage(strings.ReplaceAll(string(cs.Checkpoint), `"enum":["","A","C","B","D"]`, `"enum":["A","C","B","D",""]`)) + tb.Insert(ctx, t, tableName, [][]interface{}{{11, "A"}, {12, "B"}, {13, "C"}, {14, "D"}, {15, "error"}}) + t.Run("replication2", func(t *testing.T) { tests.VerifiedCapture(ctx, t, cs) }) +} From 75837411aa93c623908a8081e36f7aa9f7cf6015 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Tue, 12 Mar 2024 16:59:47 -0500 Subject: [PATCH 03/96] source-mysql: Migrate the old enum metadata ordering This fixes the enum decoding off-by-one bug introduced in https://github.com/estuary/connectors/pull/1336 by adding some startup logic which will migrate any old metadata to the new ordering (where `""` is the first element). Once every capture in production has run this code and had its metadata migrated it will be safe to remove all of the extra migration logic and the associated test. --- .../TestEnumDecodingFix-replication2 | 12 ++++---- source-mysql/capture_test.go | 3 ++ source-mysql/replication.go | 30 +++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/source-mysql/.snapshots/TestEnumDecodingFix-replication2 b/source-mysql/.snapshots/TestEnumDecodingFix-replication2 index da351a92c7..adcd24bedd 100644 --- a/source-mysql/.snapshots/TestEnumDecodingFix-replication2 +++ b/source-mysql/.snapshots/TestEnumDecodingFix-replication2 @@ -1,13 +1,13 @@ # ================================ # Collection "acmeCo/test/test_enumdecodingfix_32314857": 5 Documents # ================================ -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"C","id":11} -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"D","id":12} -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"B","id":13} -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"","id":14} -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"A","id":15} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"A","id":11} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"B","id":12} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"C","id":13} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"D","id":14} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"EnumDecodingFix_32314857","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"category":"","id":15} # ================================ # Final State Checkpoint # ================================ -{"bindingStateV1":{"test%2FEnumDecodingFix_32314857":{"backfilled":5,"key_columns":["id"],"metadata":{"schema":{"columns":["id","category"],"types":{"category":{"enum":["A","C","B","D",""],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} +{"bindingStateV1":{"test%2FEnumDecodingFix_32314857":{"backfilled":5,"key_columns":["id"],"metadata":{"schema":{"columns":["id","category"],"types":{"category":{"enum":["","A","C","B","D"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} diff --git a/source-mysql/capture_test.go b/source-mysql/capture_test.go index 965b1c83d9..ee3b094a2c 100644 --- a/source-mysql/capture_test.go +++ b/source-mysql/capture_test.go @@ -183,6 +183,9 @@ func TestEnumPrimaryKey(t *testing.T) { } func TestEnumDecodingFix(t *testing.T) { + // This test is part of the fix for the enum decoding bug introduced by https://github.com/estuary/connectors/pull/1336 + // and can be deleted after the metadata migration logic is also removed. The metadata migration logic can safely be + // removed after all live captures we care about have run the fix code and updated their checkpoint metadata. var tb, ctx = mysqlTestBackend(t), context.Background() var uniqueID = "32314857" var tableName = tb.CreateTable(ctx, t, uniqueID, "(id INTEGER PRIMARY KEY, category ENUM('A', 'C', 'B', 'D'))") diff --git a/source-mysql/replication.go b/source-mysql/replication.go index 6e767b9a29..46d22ba62b 100644 --- a/source-mysql/replication.go +++ b/source-mysql/replication.go @@ -769,6 +769,36 @@ func (rs *mysqlReplicationStream) ActivateTable(ctx context.Context, streamID st } } } + + // This is a temporary piece of migration logic added in March of 2024. In the PR + // https://github.com/estuary/connectors/pull/1336 the ordering of enum cases in this + // metadata was changed to simplify the decoding logic. Specifically the order of the + // cases was changed from ["A", "B", ""] to ["", "A", "B"] to more directly mirror how + // MySQL represents the illegal-enum value "" as integer 0. However this introduced a bug + // when the new indexing code was used with older metadata from before that change, and + // by the time this was discovered there were captures in production with the new ordering + // in their checkpointed metadata, so a simple revert was not an option. + // + // The solution is to push forward and add the missing migration step, so that upon table + // activation any metadata containing the old ordering will be updated to the new. Once all + // metadata in production has been so updated, it will be safe to remove this logic and the + // associated 'TestEnumDecodingFix' test case. + for column, columnType := range metadata.Schema.ColumnTypes { + if enumType, ok := columnType.(*mysqlColumnType); ok && enumType.Type == "enum" { + if len(enumType.EnumValues) > 0 && enumType.EnumValues[0] != "" { + var logEntry = logrus.WithField("column", column) + var vals = enumType.EnumValues + logEntry.WithField("vals", vals).Warn("old enum metadata ordering detected, will migrate") + if vals[len(vals)-1] == "" { + logEntry.Info("trimming empty-string case from the end") + vals = vals[:len(vals)-1] + } + vals = append([]string{""}, vals...) + logEntry.WithField("vals", vals).Info("migrated old enum metadata") + enumType.EnumValues = vals + } + } + } } else if discovery != nil { // If metadata JSON is not present, construct new default metadata based on the discovery info. logrus.WithField("stream", streamID).Debug("initializing table metadata") From 11a6e29b7ed9deb1939806f9a1e2ae95e48fb792 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 1 Mar 2024 13:43:50 +0000 Subject: [PATCH 04/96] materialize-databricks: use PUT api for uploading files --- materialize-databricks/driver.go | 4 ++-- materialize-databricks/staged_file.go | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/materialize-databricks/driver.go b/materialize-databricks/driver.go index 99820500cd..ec6e67eb29 100644 --- a/materialize-databricks/driver.go +++ b/materialize-databricks/driver.go @@ -246,8 +246,8 @@ func (t *transactor) addBinding(ctx context.Context, target sql.Table, _range *p return out } - b.loadFile = newStagedFile(t.wsClient.Files, b.rootStagingPath, translatedFieldNames(target.KeyNames())) - b.storeFile = newStagedFile(t.wsClient.Files, b.rootStagingPath, translatedFieldNames(target.ColumnNames())) + b.loadFile = newStagedFile(t.store.conn, b.rootStagingPath, translatedFieldNames(target.KeyNames())) + b.storeFile = newStagedFile(t.store.conn, b.rootStagingPath, translatedFieldNames(target.ColumnNames())) t.bindings = append(t.bindings, b) diff --git a/materialize-databricks/staged_file.go b/materialize-databricks/staged_file.go index e8e952f618..7176d570c3 100644 --- a/materialize-databricks/staged_file.go +++ b/materialize-databricks/staged_file.go @@ -7,7 +7,8 @@ import ( "os" "path/filepath" - "github.com/databricks/databricks-sdk-go/service/files" + stdsql "database/sql" + driverctx "github.com/databricks/databricks-sql-go/driverctx" sql "github.com/estuary/connectors/materialize-sql" "github.com/google/uuid" log "github.com/sirupsen/logrus" @@ -73,12 +74,12 @@ type stagedFile struct { // The remote root directory for uploading files root string - filesAPI *files.FilesAPI - // Indicates if the stagedFile has been initialized for this transaction yet. Set `true` by // start() and `false` by flush(). started bool + conn *stdsql.Conn + // References to the current file being written. buf *fileBuffer encoder *sql.CountingEncoder @@ -93,15 +94,15 @@ type stagedFile struct { groupCtx context.Context // Used to check for group cancellation upon the worker returning an error. } -func newStagedFile(filesAPI *files.FilesAPI, root string, fields []string) *stagedFile { +func newStagedFile(conn *stdsql.Conn, root string, fields []string) *stagedFile { uuid := uuid.NewString() var tempdir = os.TempDir() return &stagedFile{ - fields: fields, - dir: filepath.Join(tempdir, uuid), - root: root, - filesAPI: filesAPI, + fields: fields, + dir: filepath.Join(tempdir, uuid), + root: root, + conn: conn, } } @@ -191,10 +192,10 @@ func (f *stagedFile) putWorker(ctx context.Context, filePaths <-chan string) err var fName = filepath.Base(file) log.WithField("filepath", f.remoteFilePath(fName)).Debug("staged file: uploading") - if r, err := os.Open(file); err != nil { - return fmt.Errorf("opening file: %w", err) - } else if err := f.filesAPI.Upload(ctx, files.UploadRequest{Contents: r, FilePath: f.remoteFilePath(fName)}); err != nil { - return fmt.Errorf("uploading file: %w", err) + + ctx = driverctx.NewContextWithStagingInfo(ctx, []string{f.dir}) + if _, err := f.conn.ExecContext(ctx, fmt.Sprintf(`PUT '%s' INTO '%s'`, file, f.remoteFilePath(fName))); err != nil { + return fmt.Errorf("put file: %w", err) } log.WithField("filepath", f.remoteFilePath(fName)).Debug("staged file: upload done") From 7c685b56993dd5de4fd5476cef0e1ff269f1f5f6 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 1 Mar 2024 13:46:53 +0000 Subject: [PATCH 05/96] materialize-databricks: remove vestigial driverctx usage --- materialize-databricks/driver.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/materialize-databricks/driver.go b/materialize-databricks/driver.go index ec6e67eb29..59bc74e689 100644 --- a/materialize-databricks/driver.go +++ b/materialize-databricks/driver.go @@ -15,7 +15,6 @@ import ( "github.com/databricks/databricks-sdk-go" dbConfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/logger" - driverctx "github.com/databricks/databricks-sql-go/driverctx" dbsqllog "github.com/databricks/databricks-sql-go/logger" m "github.com/estuary/connectors/go/protocols/materialize" boilerplate "github.com/estuary/connectors/materialize-boilerplate" @@ -127,10 +126,6 @@ type transactor struct { updateDelay time.Duration } -func (t *transactor) Context(ctx context.Context) context.Context { - return driverctx.NewContextWithStagingInfo(ctx, []string{t.localStagingPath}) -} - func (d *transactor) UnmarshalState(state json.RawMessage) error { // A connector running on the "old" state may not have emitted an ack yet. We don't support // migrating states so hopefully this is nothing but an empty checkpoint. @@ -259,7 +254,7 @@ func (t *transactor) AckDelay() time.Duration { } func (d *transactor) Load(it *m.LoadIterator, loaded func(int, json.RawMessage) error) error { - var ctx = d.Context(it.Context()) + var ctx = it.Context() log.Info("load: starting upload and copying of files") for it.Next() { From 9bc35a028569cda7d77527713d368862d2080f00 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 13 Mar 2024 11:26:48 +0000 Subject: [PATCH 06/96] materialize-databricks: use PUT OVERWRITE to bypass db file upload bug --- materialize-databricks/staged_file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/materialize-databricks/staged_file.go b/materialize-databricks/staged_file.go index 7176d570c3..fd948ddab2 100644 --- a/materialize-databricks/staged_file.go +++ b/materialize-databricks/staged_file.go @@ -194,7 +194,7 @@ func (f *stagedFile) putWorker(ctx context.Context, filePaths <-chan string) err log.WithField("filepath", f.remoteFilePath(fName)).Debug("staged file: uploading") ctx = driverctx.NewContextWithStagingInfo(ctx, []string{f.dir}) - if _, err := f.conn.ExecContext(ctx, fmt.Sprintf(`PUT '%s' INTO '%s'`, file, f.remoteFilePath(fName))); err != nil { + if _, err := f.conn.ExecContext(ctx, fmt.Sprintf(`PUT '%s' INTO '%s' OVERWRITE`, file, f.remoteFilePath(fName))); err != nil { return fmt.Errorf("put file: %w", err) } log.WithField("filepath", f.remoteFilePath(fName)).Debug("staged file: upload done") From c649770b767064c961d5f60ecaf38bc08b2fcda4 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 13 Mar 2024 11:34:05 +0000 Subject: [PATCH 07/96] materialize-databricks: use short-lived connection for PUT --- materialize-databricks/driver.go | 4 ++-- materialize-databricks/staged_file.go | 15 +++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/materialize-databricks/driver.go b/materialize-databricks/driver.go index 59bc74e689..bb95b68466 100644 --- a/materialize-databricks/driver.go +++ b/materialize-databricks/driver.go @@ -241,8 +241,8 @@ func (t *transactor) addBinding(ctx context.Context, target sql.Table, _range *p return out } - b.loadFile = newStagedFile(t.store.conn, b.rootStagingPath, translatedFieldNames(target.KeyNames())) - b.storeFile = newStagedFile(t.store.conn, b.rootStagingPath, translatedFieldNames(target.ColumnNames())) + b.loadFile = newStagedFile(t.cfg, b.rootStagingPath, translatedFieldNames(target.KeyNames())) + b.storeFile = newStagedFile(t.cfg, b.rootStagingPath, translatedFieldNames(target.ColumnNames())) t.bindings = append(t.bindings, b) diff --git a/materialize-databricks/staged_file.go b/materialize-databricks/staged_file.go index fd948ddab2..01f1a28df3 100644 --- a/materialize-databricks/staged_file.go +++ b/materialize-databricks/staged_file.go @@ -78,7 +78,7 @@ type stagedFile struct { // start() and `false` by flush(). started bool - conn *stdsql.Conn + cfg *config // References to the current file being written. buf *fileBuffer @@ -94,7 +94,7 @@ type stagedFile struct { groupCtx context.Context // Used to check for group cancellation upon the worker returning an error. } -func newStagedFile(conn *stdsql.Conn, root string, fields []string) *stagedFile { +func newStagedFile(cfg *config, root string, fields []string) *stagedFile { uuid := uuid.NewString() var tempdir = os.TempDir() @@ -102,7 +102,7 @@ func newStagedFile(conn *stdsql.Conn, root string, fields []string) *stagedFile fields: fields, dir: filepath.Join(tempdir, uuid), root: root, - conn: conn, + cfg: cfg, } } @@ -193,8 +193,15 @@ func (f *stagedFile) putWorker(ctx context.Context, filePaths <-chan string) err var fName = filepath.Base(file) log.WithField("filepath", f.remoteFilePath(fName)).Debug("staged file: uploading") + db, err := stdsql.Open("databricks", f.cfg.ToURI()) + if err != nil { + return fmt.Errorf("sql.Open: %w", err) + } + defer db.Close() + ctx = driverctx.NewContextWithStagingInfo(ctx, []string{f.dir}) - if _, err := f.conn.ExecContext(ctx, fmt.Sprintf(`PUT '%s' INTO '%s' OVERWRITE`, file, f.remoteFilePath(fName))); err != nil { + + if _, err := db.ExecContext(ctx, fmt.Sprintf(`PUT '%s' INTO '%s' OVERWRITE`, file, f.remoteFilePath(fName))); err != nil { return fmt.Errorf("put file: %w", err) } log.WithField("filepath", f.remoteFilePath(fName)).Debug("staged file: upload done") From 082a175b5b9fa167a2b6edbeb6fe70d89c96bf9f Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 8 Mar 2024 15:15:36 +0000 Subject: [PATCH 08/96] materialize-snowflake: create schema if doesn't exist --- materialize-snowflake/client.go | 10 +- materialize-snowflake/config.go | 13 +- materialize-snowflake/pipe.go | 2 +- materialize-snowflake/snowflake.go | 12 +- .../materialize-snowflake/snapshot.json | 116 +++++++++--------- 5 files changed, 81 insertions(+), 72 deletions(-) diff --git a/materialize-snowflake/client.go b/materialize-snowflake/client.go index 758bee5325..bbbe15042b 100644 --- a/materialize-snowflake/client.go +++ b/materialize-snowflake/client.go @@ -22,7 +22,7 @@ type client struct { func newClient(ctx context.Context, ep *sql.Endpoint) (sql.Client, error) { cfg := ep.Config.(*config) - db, err := stdsql.Open("snowflake", cfg.ToURI(ep.Tenant)) + db, err := stdsql.Open("snowflake", cfg.ToURI(ep.Tenant, false)) if err != nil { return nil, err } @@ -56,6 +56,12 @@ func (c *client) PutSpec(ctx context.Context, updateSpec sql.MetaSpecsUpdate) er } func (c *client) CreateTable(ctx context.Context, tc sql.TableCreate) error { + if len(tc.Path) > 1 { + var schemaName = tc.Path[0] + if _, err := c.db.ExecContext(ctx, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %q", schemaName)); err != nil { + return err + } + } _, err := c.db.ExecContext(ctx, tc.TableCreateSql) return err } @@ -99,7 +105,7 @@ func (c *client) PreReqs(ctx context.Context) *sql.PrereqErr { case 390100: err = fmt.Errorf("incorrect username or password") case 390201: - // This means "doesn't exist or not authorized", and we don't have a way to + // This means "doesn't exist or not authorized", and we don't have a great way to // distinguish between that for the database, schema, or warehouse. The snowflake // error message in these cases is fairly decent fortunately. case 390189: diff --git a/materialize-snowflake/config.go b/materialize-snowflake/config.go index 84169ddf91..47834d290a 100644 --- a/materialize-snowflake/config.go +++ b/materialize-snowflake/config.go @@ -40,9 +40,9 @@ type advancedConfig struct { } // ToURI converts the Config to a DSN string. -func (c *config) ToURI(tenant string) string { +func (c *config) ToURI(tenant string, withSchema bool) string { // Build a DSN connection string. - var configCopy = c.asSnowflakeConfig(tenant) + var configCopy = c.asSnowflakeConfig(tenant, withSchema) // client_session_keep_alive causes the driver to issue a periodic keepalive request. // Without this, the authentication token will expire after 4 hours of inactivity. // The Params map will not have been initialized if the endpoint config didn't specify @@ -75,15 +75,20 @@ func (c *credentialConfig) privateKey() (*rsa.PrivateKey, error) { return nil, fmt.Errorf("only supported with JWT authentication") } -func (c *config) asSnowflakeConfig(tenant string) sf.Config { +func (c *config) asSnowflakeConfig(tenant string, withSchema bool) sf.Config { var maxStatementCount string = "0" var json string = "json" + var schema = "" + if withSchema { + schema = c.Schema + } + var conf = sf.Config{ Account: c.Account, Host: c.Host, Database: c.Database, - Schema: c.Schema, + Schema: schema, Warehouse: c.Warehouse, Role: c.Role, Application: fmt.Sprintf("%s_EstuaryFlow", tenant), diff --git a/materialize-snowflake/pipe.go b/materialize-snowflake/pipe.go index 1775df0f8a..a3ea34eb3c 100644 --- a/materialize-snowflake/pipe.go +++ b/materialize-snowflake/pipe.go @@ -72,7 +72,7 @@ func generateJWTToken(key *rsa.PrivateKey, user string, accountName string) (str func NewPipeClient(cfg *config, accountName string, tenant string) (*PipeClient, error) { httpClient := http.Client{} - var dsn = cfg.ToURI(tenant) + var dsn = cfg.ToURI(tenant, true) dsnURL, err := url.Parse(fmt.Sprintf("https://%s", dsn)) if err != nil { return nil, fmt.Errorf("parsing snowflake dsn: %w", err) diff --git a/materialize-snowflake/snowflake.go b/materialize-snowflake/snowflake.go index e8cbf84005..2d7c097e46 100644 --- a/materialize-snowflake/snowflake.go +++ b/materialize-snowflake/snowflake.go @@ -62,9 +62,6 @@ func schemasEqual(s1 string, s2 string) bool { } func (c tableConfig) Path() sql.TablePath { - if c.Schema == "" || schemasEqual(c.Schema, c.endpointSchema) { - return []string{c.Table} - } return []string{c.Schema, c.Table} } @@ -95,7 +92,7 @@ func newSnowflakeDriver() *sql.Driver { "tenant": tenant, }).Info("opening Snowflake") - var metaBase sql.TablePath + var metaBase sql.TablePath = []string{parsed.Schema} var metaSpecs, _ = sql.MetaTables(metaBase) var dialect = snowflakeDialect(parsed.Schema) @@ -185,7 +182,7 @@ func newTransactor( dialect := snowflakeDialect(cfg.Schema) - db, err := stdsql.Open("snowflake", cfg.ToURI(ep.Tenant)) + db, err := stdsql.Open("snowflake", cfg.ToURI(ep.Tenant, true)) if err != nil { return nil, fmt.Errorf("newTransactor stdsql.Open: %w", err) } @@ -218,12 +215,13 @@ func newTransactor( d.store.fence = &fence // Establish connections. - if db, err := stdsql.Open("snowflake", cfg.ToURI(ep.Tenant)); err != nil { + if db, err := stdsql.Open("snowflake", cfg.ToURI(ep.Tenant, true)); err != nil { return nil, fmt.Errorf("load stdsql.Open: %w", err) } else if d.load.conn, err = db.Conn(ctx); err != nil { return nil, fmt.Errorf("load db.Conn: %w", err) } - if db, err := stdsql.Open("snowflake", cfg.ToURI(ep.Tenant)); err != nil { + + if db, err := stdsql.Open("snowflake", cfg.ToURI(ep.Tenant, true)); err != nil { return nil, fmt.Errorf("store stdsql.Open: %w", err) } else if d.store.conn, err = db.Conn(ctx); err != nil { return nil, fmt.Errorf("store db.Conn: %w", err) diff --git a/tests/materialize/materialize-snowflake/snapshot.json b/tests/materialize/materialize-snowflake/snapshot.json index 8decda6fea..ebb1e34095 100644 --- a/tests/materialize/materialize-snowflake/snapshot.json +++ b/tests/materialize/materialize-snowflake/snapshot.json @@ -1,6 +1,6 @@ { "applied": { - "actionDescription": "\nCREATE TABLE IF NOT EXISTS simple (\n\tid INTEGER NOT NULL,\n\tcanary STRING NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE simple IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/simple';\nCOMMENT ON COLUMN simple.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN simple.canary IS 'auto-generated projection of JSON at: /canary with inferred types: [string]';\nCOMMENT ON COLUMN simple.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN simple.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS duplicate_keys_standard (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE duplicate_keys_standard IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN duplicate_keys_standard.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_standard.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN duplicate_keys_standard.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_standard.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\nCOMMENT ON COLUMN duplicate_keys_standard.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS duplicate_keys_delta (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL,\n\tflow_document VARIANT NOT NULL\n);\n\nCOMMENT ON TABLE duplicate_keys_delta IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN duplicate_keys_delta.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_delta.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN duplicate_keys_delta.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_delta.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\nCOMMENT ON COLUMN duplicate_keys_delta.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS duplicate_keys_delta_exclude_flow_doc (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL\n);\n\nCOMMENT ON TABLE duplicate_keys_delta_exclude_flow_doc IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN duplicate_keys_delta_exclude_flow_doc.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_delta_exclude_flow_doc.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN duplicate_keys_delta_exclude_flow_doc.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_delta_exclude_flow_doc.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\n\n\nCREATE TABLE IF NOT EXISTS multiple_types (\n\tid INTEGER NOT NULL,\n\tarray_int VARIANT,\n\tbool_field BOOLEAN,\n\tfloat_field DOUBLE,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tmultiple VARIANT,\n\tnested VARIANT,\n\tnullable_int INTEGER,\n\tstr_field STRING NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE multiple_types IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/multiple-data-types';\nCOMMENT ON COLUMN multiple_types.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN multiple_types.array_int IS 'auto-generated projection of JSON at: /array_int with inferred types: [array]';\nCOMMENT ON COLUMN multiple_types.bool_field IS 'auto-generated projection of JSON at: /bool_field with inferred types: [boolean]';\nCOMMENT ON COLUMN multiple_types.float_field IS 'auto-generated projection of JSON at: /float_field with inferred types: [number]';\nCOMMENT ON COLUMN multiple_types.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN multiple_types.multiple IS 'auto-generated projection of JSON at: /multiple with inferred types: [array boolean null number object string]';\nCOMMENT ON COLUMN multiple_types.nested IS 'auto-generated projection of JSON at: /nested with inferred types: [object]';\nCOMMENT ON COLUMN multiple_types.nullable_int IS 'auto-generated projection of JSON at: /nullable_int with inferred types: [integer null]';\nCOMMENT ON COLUMN multiple_types.str_field IS 'auto-generated projection of JSON at: /str_field with inferred types: [string]';\nCOMMENT ON COLUMN multiple_types.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS formatted_strings (\n\tid INTEGER NOT NULL,\n\tdate DATE,\n\tdatetime TIMESTAMP,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint_and_str INTEGER,\n\tint_str INTEGER,\n\tnum_and_str DOUBLE,\n\tnum_str DOUBLE,\n\ttime STRING,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE formatted_strings IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/formatted-strings';\nCOMMENT ON COLUMN formatted_strings.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN formatted_strings.date IS 'auto-generated projection of JSON at: /date with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.datetime IS 'auto-generated projection of JSON at: /datetime with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.int_and_str IS 'auto-generated projection of JSON at: /int_and_str with inferred types: [integer string]';\nCOMMENT ON COLUMN formatted_strings.int_str IS 'auto-generated projection of JSON at: /int_str with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.num_and_str IS 'auto-generated projection of JSON at: /num_and_str with inferred types: [number string]';\nCOMMENT ON COLUMN formatted_strings.num_str IS 'auto-generated projection of JSON at: /num_str with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.time IS 'auto-generated projection of JSON at: /time with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS symbols (\n\t\"testing (%s)\" STRING NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tid STRING,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (\"testing (%s)\")\n);\n\nCOMMENT ON TABLE symbols IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/symbols';\nCOMMENT ON COLUMN symbols.\"testing (%s)\" IS 'auto-generated projection of JSON at: /testing (%s) with inferred types: [string]';\nCOMMENT ON COLUMN symbols.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN symbols.id IS 'auto-generated projection of JSON at: /id with inferred types: [string]';\nCOMMENT ON COLUMN symbols.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\nINSERT INTO flow_materializations_v2 (version, spec, materialization) VALUES ('test', '(a-base64-encoded-value)', 'tests/materialize-snowflake/materialize');" + "actionDescription": "\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.simple (\n\tid INTEGER NOT NULL,\n\tcanary STRING NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.simple IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/simple';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.simple.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.simple.canary IS 'auto-generated projection of JSON at: /canary with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.simple.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.simple.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.duplicate_keys_standard (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.duplicate_keys_standard IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_standard.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_standard.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_standard.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_standard.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_standard.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.duplicate_keys_delta (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL,\n\tflow_document VARIANT NOT NULL\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.duplicate_keys_delta IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.multiple_types (\n\tid INTEGER NOT NULL,\n\tarray_int VARIANT,\n\tbool_field BOOLEAN,\n\tfloat_field DOUBLE,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tmultiple VARIANT,\n\tnested VARIANT,\n\tnullable_int INTEGER,\n\tstr_field STRING NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.multiple_types IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/multiple-data-types';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.array_int IS 'auto-generated projection of JSON at: /array_int with inferred types: [array]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.bool_field IS 'auto-generated projection of JSON at: /bool_field with inferred types: [boolean]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.float_field IS 'auto-generated projection of JSON at: /float_field with inferred types: [number]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.multiple IS 'auto-generated projection of JSON at: /multiple with inferred types: [array boolean null number object string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.nested IS 'auto-generated projection of JSON at: /nested with inferred types: [object]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.nullable_int IS 'auto-generated projection of JSON at: /nullable_int with inferred types: [integer null]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.str_field IS 'auto-generated projection of JSON at: /str_field with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.formatted_strings (\n\tid INTEGER NOT NULL,\n\tdate DATE,\n\tdatetime TIMESTAMP,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint_and_str INTEGER,\n\tint_str INTEGER,\n\tnum_and_str DOUBLE,\n\tnum_str DOUBLE,\n\ttime STRING,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.formatted_strings IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/formatted-strings';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.date IS 'auto-generated projection of JSON at: /date with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.datetime IS 'auto-generated projection of JSON at: /datetime with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.int_and_str IS 'auto-generated projection of JSON at: /int_and_str with inferred types: [integer string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.int_str IS 'auto-generated projection of JSON at: /int_str with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.num_and_str IS 'auto-generated projection of JSON at: /num_and_str with inferred types: [number string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.num_str IS 'auto-generated projection of JSON at: /num_str with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.time IS 'auto-generated projection of JSON at: /time with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.symbols (\n\t\"testing (%s)\" STRING NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tid STRING,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (\"testing (%s)\")\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.symbols IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/symbols';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.symbols.\"testing (%s)\" IS 'auto-generated projection of JSON at: /testing (%s) with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.symbols.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.symbols.id IS 'auto-generated projection of JSON at: /id with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.symbols.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\nINSERT INTO ESTUARY_SCHEMA.flow_materializations_v2 (version, spec, materialization) VALUES ('test', '(a-base64-encoded-value)', 'tests/materialize-snowflake/materialize');" } } { @@ -11,13 +11,13 @@ "state": { "mergePatch": true, "updated": { - "duplicate_keys_delta": null, - "duplicate_keys_delta_exclude_flow_doc": null, - "duplicate_keys_standard": null, - "formatted_strings": null, - "multiple_types": null, - "simple": null, - "symbols": null + "ESTUARY_SCHEMA%2Fduplicate_keys_delta": null, + "ESTUARY_SCHEMA%2Fduplicate_keys_delta_exclude_flow_doc": null, + "ESTUARY_SCHEMA%2Fduplicate_keys_standard": null, + "ESTUARY_SCHEMA%2Fformatted_strings": null, + "ESTUARY_SCHEMA%2Fmultiple_types": null, + "ESTUARY_SCHEMA%2Fsimple": null, + "ESTUARY_SCHEMA%2Fsymbols": null } } } @@ -30,7 +30,7 @@ "state": { "mergePatch": true, "updated": { - "duplicate_keys_delta": { + "ESTUARY_SCHEMA%2Fduplicate_keys_delta": { "PipeFiles": [ { "Path": "", @@ -40,9 +40,9 @@ "PipeName": "MAHDI_DB.ESTUARY_SCHEMA.FLOW_PIPE_2_DUPLICATE_KEYS_DELTA_00000000", "Query": "", "StagedDir": "", - "Table": "duplicate_keys_delta" + "Table": "ESTUARY_SCHEMA.duplicate_keys_delta" }, - "duplicate_keys_delta_exclude_flow_doc": { + "ESTUARY_SCHEMA%2Fduplicate_keys_delta_exclude_flow_doc": { "PipeFiles": [ { "Path": "", @@ -52,42 +52,42 @@ "PipeName": "MAHDI_DB.ESTUARY_SCHEMA.FLOW_PIPE_3_DUPLICATE_KEYS_DELTA_EXCLUDE_FLOW_DOC_00000000", "Query": "", "StagedDir": "", - "Table": "duplicate_keys_delta_exclude_flow_doc" + "Table": "ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc" }, - "duplicate_keys_standard": { + "ESTUARY_SCHEMA%2Fduplicate_keys_standard": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO duplicate_keys_standard (\n\tid, flow_published_at, int, str, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS flow_published_at, $1[2] AS int, $1[3] AS str, $1[4] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO ESTUARY_SCHEMA.duplicate_keys_standard (\n\tid, flow_published_at, int, str, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS flow_published_at, $1[2] AS int, $1[3] AS str, $1[4] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "duplicate_keys_standard" + "Table": "ESTUARY_SCHEMA.duplicate_keys_standard" }, - "formatted_strings": { + "ESTUARY_SCHEMA%2Fformatted_strings": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO formatted_strings (\n\tid, date, datetime, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS date, $1[2] AS datetime, $1[3] AS flow_published_at, $1[4] AS int_and_str, $1[5] AS int_str, $1[6] AS num_and_str, $1[7] AS num_str, $1[8] AS time, $1[9] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO ESTUARY_SCHEMA.formatted_strings (\n\tid, date, datetime, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS date, $1[2] AS datetime, $1[3] AS flow_published_at, $1[4] AS int_and_str, $1[5] AS int_str, $1[6] AS num_and_str, $1[7] AS num_str, $1[8] AS time, $1[9] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "formatted_strings" + "Table": "ESTUARY_SCHEMA.formatted_strings" }, - "multiple_types": { + "ESTUARY_SCHEMA%2Fmultiple_types": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO multiple_types (\n\tid, array_int, bool_field, float_field, flow_published_at, multiple, nested, nullable_int, str_field, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS array_int, $1[2] AS bool_field, $1[3] AS float_field, $1[4] AS flow_published_at, $1[5] AS multiple, $1[6] AS nested, $1[7] AS nullable_int, $1[8] AS str_field, $1[9] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO ESTUARY_SCHEMA.multiple_types (\n\tid, array_int, bool_field, float_field, flow_published_at, multiple, nested, nullable_int, str_field, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS array_int, $1[2] AS bool_field, $1[3] AS float_field, $1[4] AS flow_published_at, $1[5] AS multiple, $1[6] AS nested, $1[7] AS nullable_int, $1[8] AS str_field, $1[9] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "multiple_types" + "Table": "ESTUARY_SCHEMA.multiple_types" }, - "simple": { + "ESTUARY_SCHEMA%2Fsimple": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO simple (\n\tid, canary, flow_published_at, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS canary, $1[2] AS flow_published_at, $1[3] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO ESTUARY_SCHEMA.simple (\n\tid, canary, flow_published_at, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS canary, $1[2] AS flow_published_at, $1[3] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "simple" + "Table": "ESTUARY_SCHEMA.simple" }, - "symbols": { + "ESTUARY_SCHEMA%2Fsymbols": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO symbols (\n\t\"testing (%s)\", flow_published_at, id, flow_document\n) FROM (\n\tSELECT $1[0] AS \"testing (%s)\", $1[1] AS flow_published_at, $1[2] AS id, $1[3] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO ESTUARY_SCHEMA.symbols (\n\t\"testing (%s)\", flow_published_at, id, flow_document\n) FROM (\n\tSELECT $1[0] AS \"testing (%s)\", $1[1] AS flow_published_at, $1[2] AS id, $1[3] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "symbols" + "Table": "ESTUARY_SCHEMA.symbols" } } } @@ -98,13 +98,13 @@ "state": { "mergePatch": true, "updated": { - "duplicate_keys_delta": null, - "duplicate_keys_delta_exclude_flow_doc": null, - "duplicate_keys_standard": null, - "formatted_strings": null, - "multiple_types": null, - "simple": null, - "symbols": null + "ESTUARY_SCHEMA%2Fduplicate_keys_delta": null, + "ESTUARY_SCHEMA%2Fduplicate_keys_delta_exclude_flow_doc": null, + "ESTUARY_SCHEMA%2Fduplicate_keys_standard": null, + "ESTUARY_SCHEMA%2Fformatted_strings": null, + "ESTUARY_SCHEMA%2Fmultiple_types": null, + "ESTUARY_SCHEMA%2Fsimple": null, + "ESTUARY_SCHEMA%2Fsymbols": null } } } @@ -301,7 +301,7 @@ "state": { "mergePatch": true, "updated": { - "duplicate_keys_delta": { + "ESTUARY_SCHEMA%2Fduplicate_keys_delta": { "PipeFiles": [ { "Path": "", @@ -311,9 +311,9 @@ "PipeName": "MAHDI_DB.ESTUARY_SCHEMA.FLOW_PIPE_2_DUPLICATE_KEYS_DELTA_00000000", "Query": "", "StagedDir": "", - "Table": "duplicate_keys_delta" + "Table": "ESTUARY_SCHEMA.duplicate_keys_delta" }, - "duplicate_keys_delta_exclude_flow_doc": { + "ESTUARY_SCHEMA%2Fduplicate_keys_delta_exclude_flow_doc": { "PipeFiles": [ { "Path": "", @@ -323,35 +323,35 @@ "PipeName": "MAHDI_DB.ESTUARY_SCHEMA.FLOW_PIPE_3_DUPLICATE_KEYS_DELTA_EXCLUDE_FLOW_DOC_00000000", "Query": "", "StagedDir": "", - "Table": "duplicate_keys_delta_exclude_flow_doc" + "Table": "ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc" }, - "duplicate_keys_standard": { + "ESTUARY_SCHEMA%2Fduplicate_keys_standard": { "PipeFiles": null, "PipeName": "", - "Query": "\nMERGE INTO duplicate_keys_standard AS l\nUSING (\n\tSELECT $1[0] AS id, $1[1] AS flow_published_at, $1[2] AS int, $1[3] AS str, $1[4] AS flow_document\n\tFROM \n) AS r\nON l.id = r.id\nWHEN MATCHED AND IS_NULL_VALUE(r.flow_document) THEN\n\tDELETE\nWHEN MATCHED THEN\n\tUPDATE SET l.flow_published_at = r.flow_published_at, l.int = r.int, l.str = r.str, l.flow_document = r.flow_document\nWHEN NOT MATCHED THEN\n\tINSERT (id, flow_published_at, int, str, flow_document)\n\tVALUES (r.id, r.flow_published_at, r.int, r.str, r.flow_document);\n", + "Query": "\nMERGE INTO ESTUARY_SCHEMA.duplicate_keys_standard AS l\nUSING (\n\tSELECT $1[0] AS id, $1[1] AS flow_published_at, $1[2] AS int, $1[3] AS str, $1[4] AS flow_document\n\tFROM \n) AS r\nON l.id = r.id\nWHEN MATCHED AND IS_NULL_VALUE(r.flow_document) THEN\n\tDELETE\nWHEN MATCHED THEN\n\tUPDATE SET l.flow_published_at = r.flow_published_at, l.int = r.int, l.str = r.str, l.flow_document = r.flow_document\nWHEN NOT MATCHED THEN\n\tINSERT (id, flow_published_at, int, str, flow_document)\n\tVALUES (r.id, r.flow_published_at, r.int, r.str, r.flow_document);\n", "StagedDir": "", - "Table": "duplicate_keys_standard" + "Table": "ESTUARY_SCHEMA.duplicate_keys_standard" }, - "formatted_strings": { + "ESTUARY_SCHEMA%2Fformatted_strings": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO formatted_strings (\n\tid, date, datetime, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS date, $1[2] AS datetime, $1[3] AS flow_published_at, $1[4] AS int_and_str, $1[5] AS int_str, $1[6] AS num_and_str, $1[7] AS num_str, $1[8] AS time, $1[9] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO ESTUARY_SCHEMA.formatted_strings (\n\tid, date, datetime, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS date, $1[2] AS datetime, $1[3] AS flow_published_at, $1[4] AS int_and_str, $1[5] AS int_str, $1[6] AS num_and_str, $1[7] AS num_str, $1[8] AS time, $1[9] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "formatted_strings" + "Table": "ESTUARY_SCHEMA.formatted_strings" }, - "multiple_types": { + "ESTUARY_SCHEMA%2Fmultiple_types": { "PipeFiles": null, "PipeName": "", - "Query": "\nMERGE INTO multiple_types AS l\nUSING (\n\tSELECT $1[0] AS id, $1[1] AS array_int, $1[2] AS bool_field, $1[3] AS float_field, $1[4] AS flow_published_at, $1[5] AS multiple, $1[6] AS nested, $1[7] AS nullable_int, $1[8] AS str_field, $1[9] AS flow_document\n\tFROM \n) AS r\nON l.id = r.id\nWHEN MATCHED AND IS_NULL_VALUE(r.flow_document) THEN\n\tDELETE\nWHEN MATCHED THEN\n\tUPDATE SET l.array_int = r.array_int, l.bool_field = r.bool_field, l.float_field = r.float_field, l.flow_published_at = r.flow_published_at, l.multiple = r.multiple, l.nested = r.nested, l.nullable_int = r.nullable_int, l.str_field = r.str_field, l.flow_document = r.flow_document\nWHEN NOT MATCHED THEN\n\tINSERT (id, array_int, bool_field, float_field, flow_published_at, multiple, nested, nullable_int, str_field, flow_document)\n\tVALUES (r.id, r.array_int, r.bool_field, r.float_field, r.flow_published_at, r.multiple, r.nested, r.nullable_int, r.str_field, r.flow_document);\n", + "Query": "\nMERGE INTO ESTUARY_SCHEMA.multiple_types AS l\nUSING (\n\tSELECT $1[0] AS id, $1[1] AS array_int, $1[2] AS bool_field, $1[3] AS float_field, $1[4] AS flow_published_at, $1[5] AS multiple, $1[6] AS nested, $1[7] AS nullable_int, $1[8] AS str_field, $1[9] AS flow_document\n\tFROM \n) AS r\nON l.id = r.id\nWHEN MATCHED AND IS_NULL_VALUE(r.flow_document) THEN\n\tDELETE\nWHEN MATCHED THEN\n\tUPDATE SET l.array_int = r.array_int, l.bool_field = r.bool_field, l.float_field = r.float_field, l.flow_published_at = r.flow_published_at, l.multiple = r.multiple, l.nested = r.nested, l.nullable_int = r.nullable_int, l.str_field = r.str_field, l.flow_document = r.flow_document\nWHEN NOT MATCHED THEN\n\tINSERT (id, array_int, bool_field, float_field, flow_published_at, multiple, nested, nullable_int, str_field, flow_document)\n\tVALUES (r.id, r.array_int, r.bool_field, r.float_field, r.flow_published_at, r.multiple, r.nested, r.nullable_int, r.str_field, r.flow_document);\n", "StagedDir": "", - "Table": "multiple_types" + "Table": "ESTUARY_SCHEMA.multiple_types" }, - "simple": { + "ESTUARY_SCHEMA%2Fsimple": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO simple (\n\tid, canary, flow_published_at, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS canary, $1[2] AS flow_published_at, $1[3] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO ESTUARY_SCHEMA.simple (\n\tid, canary, flow_published_at, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS canary, $1[2] AS flow_published_at, $1[3] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "simple" + "Table": "ESTUARY_SCHEMA.simple" } } } @@ -362,13 +362,13 @@ "state": { "mergePatch": true, "updated": { - "duplicate_keys_delta": null, - "duplicate_keys_delta_exclude_flow_doc": null, - "duplicate_keys_standard": null, - "formatted_strings": null, - "multiple_types": null, - "simple": null, - "symbols": null + "ESTUARY_SCHEMA%2Fduplicate_keys_delta": null, + "ESTUARY_SCHEMA%2Fduplicate_keys_delta_exclude_flow_doc": null, + "ESTUARY_SCHEMA%2Fduplicate_keys_standard": null, + "ESTUARY_SCHEMA%2Fformatted_strings": null, + "ESTUARY_SCHEMA%2Fmultiple_types": null, + "ESTUARY_SCHEMA%2Fsimple": null, + "ESTUARY_SCHEMA%2Fsymbols": null } } } @@ -857,7 +857,7 @@ } { "applied": { - "actionDescription": "UPDATE flow_materializations_v2 SET version = 'test', spec = '(a-base64-encoded-value)' WHERE materialization = 'tests/materialize-snowflake/materialize';" + "actionDescription": "UPDATE ESTUARY_SCHEMA.flow_materializations_v2 SET version = 'test', spec = '(a-base64-encoded-value)' WHERE materialization = 'tests/materialize-snowflake/materialize';" } } { From 786f5f32df3d1b413dd86fc457a2aaeb0e5664cd Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 13 Mar 2024 11:56:52 +0000 Subject: [PATCH 09/96] materialize-snowflake: backward compatibility note in tableConfig Path --- materialize-snowflake/client.go | 40 ++++-- materialize-snowflake/snowflake.go | 6 + materialize-snowflake/snowflake_test.go | 2 +- .../materialize-snowflake/snapshot.json | 114 +++++++++--------- 4 files changed, 97 insertions(+), 65 deletions(-) diff --git a/materialize-snowflake/client.go b/materialize-snowflake/client.go index bbbe15042b..1e1f210de2 100644 --- a/materialize-snowflake/client.go +++ b/materialize-snowflake/client.go @@ -56,13 +56,25 @@ func (c *client) PutSpec(ctx context.Context, updateSpec sql.MetaSpecsUpdate) er } func (c *client) CreateTable(ctx context.Context, tc sql.TableCreate) error { + var schemaName = c.cfg.Schema if len(tc.Path) > 1 { - var schemaName = tc.Path[0] - if _, err := c.db.ExecContext(ctx, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %q", schemaName)); err != nil { - return err - } + schemaName = tc.Path[0] + } + + var conn, err = c.db.Conn(ctx) + if err != nil { + return err } - _, err := c.db.ExecContext(ctx, tc.TableCreateSql) + + if _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %q", schemaName)); err != nil { + return err + } + + if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %q", c.cfg.Schema)); err != nil { + return err + } + + _, err = conn.ExecContext(ctx, tc.TableCreateSql) return err } @@ -70,7 +82,14 @@ func (c *client) DeleteTable(ctx context.Context, path []string) (string, boiler stmt := fmt.Sprintf("DROP TABLE %s;", c.ep.Dialect.Identifier(path...)) return stmt, func(ctx context.Context) error { - _, err := c.db.ExecContext(ctx, stmt) + var conn, err = c.db.Conn(ctx) + if err != nil { + return err + } + if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %q", c.cfg.Schema)); err != nil { + return err + } + _, err = conn.ExecContext(ctx, stmt) return err }, nil } @@ -83,7 +102,14 @@ func (c *client) AlterTable(ctx context.Context, ta sql.TableAlter) (string, boi alterColumnStmt := alterColumnStmtBuilder.String() return alterColumnStmt, func(ctx context.Context) error { - _, err := c.db.ExecContext(ctx, alterColumnStmt) + var conn, err = c.db.Conn(ctx) + if err != nil { + return err + } + if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %q", c.cfg.Schema)); err != nil { + return err + } + _, err = c.db.ExecContext(ctx, alterColumnStmt) return err }, nil } diff --git a/materialize-snowflake/snowflake.go b/materialize-snowflake/snowflake.go index 2d7c097e46..e2b931e047 100644 --- a/materialize-snowflake/snowflake.go +++ b/materialize-snowflake/snowflake.go @@ -62,6 +62,12 @@ func schemasEqual(s1 string, s2 string) bool { } func (c tableConfig) Path() sql.TablePath { + // This is here for backward compatibility purposes. There was a time when binding resources could not + // have schema configuration. If we change this for all bindings to be a two-part resource path, that will + // lead to a re-backfilling of the bindings which did not previously have a schema as part of their resource path + if c.Schema == "" || schemasEqual(c.Schema, c.endpointSchema) { + return []string{c.Table} + } return []string{c.Schema, c.Table} } diff --git a/materialize-snowflake/snowflake_test.go b/materialize-snowflake/snowflake_test.go index 976dd485a2..44bdf42403 100644 --- a/materialize-snowflake/snowflake_test.go +++ b/materialize-snowflake/snowflake_test.go @@ -57,7 +57,7 @@ func TestValidateAndApply(t *testing.T) { Schema: "PUBLIC", } - db, err := stdsql.Open("snowflake", cfg.ToURI("testing")) + db, err := stdsql.Open("snowflake", cfg.ToURI("testing", true)) require.NoError(t, err) defer db.Close() diff --git a/tests/materialize/materialize-snowflake/snapshot.json b/tests/materialize/materialize-snowflake/snapshot.json index ebb1e34095..8e040b9b9a 100644 --- a/tests/materialize/materialize-snowflake/snapshot.json +++ b/tests/materialize/materialize-snowflake/snapshot.json @@ -1,6 +1,6 @@ { "applied": { - "actionDescription": "\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.simple (\n\tid INTEGER NOT NULL,\n\tcanary STRING NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.simple IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/simple';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.simple.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.simple.canary IS 'auto-generated projection of JSON at: /canary with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.simple.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.simple.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.duplicate_keys_standard (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.duplicate_keys_standard IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_standard.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_standard.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_standard.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_standard.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_standard.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.duplicate_keys_delta (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL,\n\tflow_document VARIANT NOT NULL\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.duplicate_keys_delta IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.multiple_types (\n\tid INTEGER NOT NULL,\n\tarray_int VARIANT,\n\tbool_field BOOLEAN,\n\tfloat_field DOUBLE,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tmultiple VARIANT,\n\tnested VARIANT,\n\tnullable_int INTEGER,\n\tstr_field STRING NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.multiple_types IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/multiple-data-types';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.array_int IS 'auto-generated projection of JSON at: /array_int with inferred types: [array]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.bool_field IS 'auto-generated projection of JSON at: /bool_field with inferred types: [boolean]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.float_field IS 'auto-generated projection of JSON at: /float_field with inferred types: [number]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.multiple IS 'auto-generated projection of JSON at: /multiple with inferred types: [array boolean null number object string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.nested IS 'auto-generated projection of JSON at: /nested with inferred types: [object]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.nullable_int IS 'auto-generated projection of JSON at: /nullable_int with inferred types: [integer null]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.str_field IS 'auto-generated projection of JSON at: /str_field with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.multiple_types.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.formatted_strings (\n\tid INTEGER NOT NULL,\n\tdate DATE,\n\tdatetime TIMESTAMP,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint_and_str INTEGER,\n\tint_str INTEGER,\n\tnum_and_str DOUBLE,\n\tnum_str DOUBLE,\n\ttime STRING,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.formatted_strings IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/formatted-strings';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.date IS 'auto-generated projection of JSON at: /date with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.datetime IS 'auto-generated projection of JSON at: /datetime with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.int_and_str IS 'auto-generated projection of JSON at: /int_and_str with inferred types: [integer string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.int_str IS 'auto-generated projection of JSON at: /int_str with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.num_and_str IS 'auto-generated projection of JSON at: /num_and_str with inferred types: [number string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.num_str IS 'auto-generated projection of JSON at: /num_str with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.time IS 'auto-generated projection of JSON at: /time with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.formatted_strings.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS ESTUARY_SCHEMA.symbols (\n\t\"testing (%s)\" STRING NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tid STRING,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (\"testing (%s)\")\n);\n\nCOMMENT ON TABLE ESTUARY_SCHEMA.symbols IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/symbols';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.symbols.\"testing (%s)\" IS 'auto-generated projection of JSON at: /testing (%s) with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.symbols.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.symbols.id IS 'auto-generated projection of JSON at: /id with inferred types: [string]';\nCOMMENT ON COLUMN ESTUARY_SCHEMA.symbols.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\nINSERT INTO ESTUARY_SCHEMA.flow_materializations_v2 (version, spec, materialization) VALUES ('test', '(a-base64-encoded-value)', 'tests/materialize-snowflake/materialize');" + "actionDescription": "\nCREATE TABLE IF NOT EXISTS simple (\n\tid INTEGER NOT NULL,\n\tcanary STRING NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE simple IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/simple';\nCOMMENT ON COLUMN simple.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN simple.canary IS 'auto-generated projection of JSON at: /canary with inferred types: [string]';\nCOMMENT ON COLUMN simple.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN simple.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS duplicate_keys_standard (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE duplicate_keys_standard IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN duplicate_keys_standard.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_standard.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN duplicate_keys_standard.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_standard.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\nCOMMENT ON COLUMN duplicate_keys_standard.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS duplicate_keys_delta (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL,\n\tflow_document VARIANT NOT NULL\n);\n\nCOMMENT ON TABLE duplicate_keys_delta IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN duplicate_keys_delta.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_delta.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN duplicate_keys_delta.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_delta.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\nCOMMENT ON COLUMN duplicate_keys_delta.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS duplicate_keys_delta_exclude_flow_doc (\n\tid INTEGER NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint INTEGER,\n\tstr STRING NOT NULL\n);\n\nCOMMENT ON TABLE duplicate_keys_delta_exclude_flow_doc IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/duplicated-keys';\nCOMMENT ON COLUMN duplicate_keys_delta_exclude_flow_doc.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_delta_exclude_flow_doc.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN duplicate_keys_delta_exclude_flow_doc.int IS 'auto-generated projection of JSON at: /int with inferred types: [integer]';\nCOMMENT ON COLUMN duplicate_keys_delta_exclude_flow_doc.str IS 'auto-generated projection of JSON at: /str with inferred types: [string]';\n\n\nCREATE TABLE IF NOT EXISTS multiple_types (\n\tid INTEGER NOT NULL,\n\tarray_int VARIANT,\n\tbool_field BOOLEAN,\n\tfloat_field DOUBLE,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tmultiple VARIANT,\n\tnested VARIANT,\n\tnullable_int INTEGER,\n\tstr_field STRING NOT NULL,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE multiple_types IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/multiple-data-types';\nCOMMENT ON COLUMN multiple_types.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN multiple_types.array_int IS 'auto-generated projection of JSON at: /array_int with inferred types: [array]';\nCOMMENT ON COLUMN multiple_types.bool_field IS 'auto-generated projection of JSON at: /bool_field with inferred types: [boolean]';\nCOMMENT ON COLUMN multiple_types.float_field IS 'auto-generated projection of JSON at: /float_field with inferred types: [number]';\nCOMMENT ON COLUMN multiple_types.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN multiple_types.multiple IS 'auto-generated projection of JSON at: /multiple with inferred types: [array boolean null number object string]';\nCOMMENT ON COLUMN multiple_types.nested IS 'auto-generated projection of JSON at: /nested with inferred types: [object]';\nCOMMENT ON COLUMN multiple_types.nullable_int IS 'auto-generated projection of JSON at: /nullable_int with inferred types: [integer null]';\nCOMMENT ON COLUMN multiple_types.str_field IS 'auto-generated projection of JSON at: /str_field with inferred types: [string]';\nCOMMENT ON COLUMN multiple_types.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS formatted_strings (\n\tid INTEGER NOT NULL,\n\tdate DATE,\n\tdatetime TIMESTAMP,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tint_and_str INTEGER,\n\tint_str INTEGER,\n\tnum_and_str DOUBLE,\n\tnum_str DOUBLE,\n\ttime STRING,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (id)\n);\n\nCOMMENT ON TABLE formatted_strings IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/formatted-strings';\nCOMMENT ON COLUMN formatted_strings.id IS 'auto-generated projection of JSON at: /id with inferred types: [integer]';\nCOMMENT ON COLUMN formatted_strings.date IS 'auto-generated projection of JSON at: /date with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.datetime IS 'auto-generated projection of JSON at: /datetime with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.int_and_str IS 'auto-generated projection of JSON at: /int_and_str with inferred types: [integer string]';\nCOMMENT ON COLUMN formatted_strings.int_str IS 'auto-generated projection of JSON at: /int_str with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.num_and_str IS 'auto-generated projection of JSON at: /num_and_str with inferred types: [number string]';\nCOMMENT ON COLUMN formatted_strings.num_str IS 'auto-generated projection of JSON at: /num_str with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.time IS 'auto-generated projection of JSON at: /time with inferred types: [string]';\nCOMMENT ON COLUMN formatted_strings.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\n\nCREATE TABLE IF NOT EXISTS symbols (\n\t\"testing (%s)\" STRING NOT NULL,\n\tflow_published_at TIMESTAMP NOT NULL,\n\tid STRING,\n\tflow_document VARIANT NOT NULL,\n\n\tPRIMARY KEY (\"testing (%s)\")\n);\n\nCOMMENT ON TABLE symbols IS 'Generated for materialization tests/materialize-snowflake/materialize of collection tests/symbols';\nCOMMENT ON COLUMN symbols.\"testing (%s)\" IS 'auto-generated projection of JSON at: /testing (%s) with inferred types: [string]';\nCOMMENT ON COLUMN symbols.flow_published_at IS 'Flow Publication Time\nFlow publication date-time of this document\nauto-generated projection of JSON at: /_meta/uuid with inferred types: [string]';\nCOMMENT ON COLUMN symbols.id IS 'auto-generated projection of JSON at: /id with inferred types: [string]';\nCOMMENT ON COLUMN symbols.flow_document IS 'auto-generated projection of JSON at: with inferred types: [object]';\n\nINSERT INTO ESTUARY_SCHEMA.flow_materializations_v2 (version, spec, materialization) VALUES ('test', '(a-base64-encoded-value)', 'tests/materialize-snowflake/materialize');" } } { @@ -11,13 +11,13 @@ "state": { "mergePatch": true, "updated": { - "ESTUARY_SCHEMA%2Fduplicate_keys_delta": null, - "ESTUARY_SCHEMA%2Fduplicate_keys_delta_exclude_flow_doc": null, - "ESTUARY_SCHEMA%2Fduplicate_keys_standard": null, - "ESTUARY_SCHEMA%2Fformatted_strings": null, - "ESTUARY_SCHEMA%2Fmultiple_types": null, - "ESTUARY_SCHEMA%2Fsimple": null, - "ESTUARY_SCHEMA%2Fsymbols": null + "duplicate_keys_delta": null, + "duplicate_keys_delta_exclude_flow_doc": null, + "duplicate_keys_standard": null, + "formatted_strings": null, + "multiple_types": null, + "simple": null, + "symbols": null } } } @@ -30,7 +30,7 @@ "state": { "mergePatch": true, "updated": { - "ESTUARY_SCHEMA%2Fduplicate_keys_delta": { + "duplicate_keys_delta": { "PipeFiles": [ { "Path": "", @@ -40,9 +40,9 @@ "PipeName": "MAHDI_DB.ESTUARY_SCHEMA.FLOW_PIPE_2_DUPLICATE_KEYS_DELTA_00000000", "Query": "", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.duplicate_keys_delta" + "Table": "duplicate_keys_delta" }, - "ESTUARY_SCHEMA%2Fduplicate_keys_delta_exclude_flow_doc": { + "duplicate_keys_delta_exclude_flow_doc": { "PipeFiles": [ { "Path": "", @@ -52,42 +52,42 @@ "PipeName": "MAHDI_DB.ESTUARY_SCHEMA.FLOW_PIPE_3_DUPLICATE_KEYS_DELTA_EXCLUDE_FLOW_DOC_00000000", "Query": "", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc" + "Table": "duplicate_keys_delta_exclude_flow_doc" }, - "ESTUARY_SCHEMA%2Fduplicate_keys_standard": { + "duplicate_keys_standard": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO ESTUARY_SCHEMA.duplicate_keys_standard (\n\tid, flow_published_at, int, str, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS flow_published_at, $1[2] AS int, $1[3] AS str, $1[4] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO duplicate_keys_standard (\n\tid, flow_published_at, int, str, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS flow_published_at, $1[2] AS int, $1[3] AS str, $1[4] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.duplicate_keys_standard" + "Table": "duplicate_keys_standard" }, - "ESTUARY_SCHEMA%2Fformatted_strings": { + "formatted_strings": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO ESTUARY_SCHEMA.formatted_strings (\n\tid, date, datetime, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS date, $1[2] AS datetime, $1[3] AS flow_published_at, $1[4] AS int_and_str, $1[5] AS int_str, $1[6] AS num_and_str, $1[7] AS num_str, $1[8] AS time, $1[9] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO formatted_strings (\n\tid, date, datetime, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS date, $1[2] AS datetime, $1[3] AS flow_published_at, $1[4] AS int_and_str, $1[5] AS int_str, $1[6] AS num_and_str, $1[7] AS num_str, $1[8] AS time, $1[9] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.formatted_strings" + "Table": "formatted_strings" }, - "ESTUARY_SCHEMA%2Fmultiple_types": { + "multiple_types": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO ESTUARY_SCHEMA.multiple_types (\n\tid, array_int, bool_field, float_field, flow_published_at, multiple, nested, nullable_int, str_field, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS array_int, $1[2] AS bool_field, $1[3] AS float_field, $1[4] AS flow_published_at, $1[5] AS multiple, $1[6] AS nested, $1[7] AS nullable_int, $1[8] AS str_field, $1[9] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO multiple_types (\n\tid, array_int, bool_field, float_field, flow_published_at, multiple, nested, nullable_int, str_field, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS array_int, $1[2] AS bool_field, $1[3] AS float_field, $1[4] AS flow_published_at, $1[5] AS multiple, $1[6] AS nested, $1[7] AS nullable_int, $1[8] AS str_field, $1[9] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.multiple_types" + "Table": "multiple_types" }, - "ESTUARY_SCHEMA%2Fsimple": { + "simple": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO ESTUARY_SCHEMA.simple (\n\tid, canary, flow_published_at, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS canary, $1[2] AS flow_published_at, $1[3] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO simple (\n\tid, canary, flow_published_at, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS canary, $1[2] AS flow_published_at, $1[3] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.simple" + "Table": "simple" }, - "ESTUARY_SCHEMA%2Fsymbols": { + "symbols": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO ESTUARY_SCHEMA.symbols (\n\t\"testing (%s)\", flow_published_at, id, flow_document\n) FROM (\n\tSELECT $1[0] AS \"testing (%s)\", $1[1] AS flow_published_at, $1[2] AS id, $1[3] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO symbols (\n\t\"testing (%s)\", flow_published_at, id, flow_document\n) FROM (\n\tSELECT $1[0] AS \"testing (%s)\", $1[1] AS flow_published_at, $1[2] AS id, $1[3] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.symbols" + "Table": "symbols" } } } @@ -98,13 +98,13 @@ "state": { "mergePatch": true, "updated": { - "ESTUARY_SCHEMA%2Fduplicate_keys_delta": null, - "ESTUARY_SCHEMA%2Fduplicate_keys_delta_exclude_flow_doc": null, - "ESTUARY_SCHEMA%2Fduplicate_keys_standard": null, - "ESTUARY_SCHEMA%2Fformatted_strings": null, - "ESTUARY_SCHEMA%2Fmultiple_types": null, - "ESTUARY_SCHEMA%2Fsimple": null, - "ESTUARY_SCHEMA%2Fsymbols": null + "duplicate_keys_delta": null, + "duplicate_keys_delta_exclude_flow_doc": null, + "duplicate_keys_standard": null, + "formatted_strings": null, + "multiple_types": null, + "simple": null, + "symbols": null } } } @@ -301,7 +301,7 @@ "state": { "mergePatch": true, "updated": { - "ESTUARY_SCHEMA%2Fduplicate_keys_delta": { + "duplicate_keys_delta": { "PipeFiles": [ { "Path": "", @@ -311,9 +311,9 @@ "PipeName": "MAHDI_DB.ESTUARY_SCHEMA.FLOW_PIPE_2_DUPLICATE_KEYS_DELTA_00000000", "Query": "", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.duplicate_keys_delta" + "Table": "duplicate_keys_delta" }, - "ESTUARY_SCHEMA%2Fduplicate_keys_delta_exclude_flow_doc": { + "duplicate_keys_delta_exclude_flow_doc": { "PipeFiles": [ { "Path": "", @@ -323,35 +323,35 @@ "PipeName": "MAHDI_DB.ESTUARY_SCHEMA.FLOW_PIPE_3_DUPLICATE_KEYS_DELTA_EXCLUDE_FLOW_DOC_00000000", "Query": "", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.duplicate_keys_delta_exclude_flow_doc" + "Table": "duplicate_keys_delta_exclude_flow_doc" }, - "ESTUARY_SCHEMA%2Fduplicate_keys_standard": { + "duplicate_keys_standard": { "PipeFiles": null, "PipeName": "", - "Query": "\nMERGE INTO ESTUARY_SCHEMA.duplicate_keys_standard AS l\nUSING (\n\tSELECT $1[0] AS id, $1[1] AS flow_published_at, $1[2] AS int, $1[3] AS str, $1[4] AS flow_document\n\tFROM \n) AS r\nON l.id = r.id\nWHEN MATCHED AND IS_NULL_VALUE(r.flow_document) THEN\n\tDELETE\nWHEN MATCHED THEN\n\tUPDATE SET l.flow_published_at = r.flow_published_at, l.int = r.int, l.str = r.str, l.flow_document = r.flow_document\nWHEN NOT MATCHED THEN\n\tINSERT (id, flow_published_at, int, str, flow_document)\n\tVALUES (r.id, r.flow_published_at, r.int, r.str, r.flow_document);\n", + "Query": "\nMERGE INTO duplicate_keys_standard AS l\nUSING (\n\tSELECT $1[0] AS id, $1[1] AS flow_published_at, $1[2] AS int, $1[3] AS str, $1[4] AS flow_document\n\tFROM \n) AS r\nON l.id = r.id\nWHEN MATCHED AND IS_NULL_VALUE(r.flow_document) THEN\n\tDELETE\nWHEN MATCHED THEN\n\tUPDATE SET l.flow_published_at = r.flow_published_at, l.int = r.int, l.str = r.str, l.flow_document = r.flow_document\nWHEN NOT MATCHED THEN\n\tINSERT (id, flow_published_at, int, str, flow_document)\n\tVALUES (r.id, r.flow_published_at, r.int, r.str, r.flow_document);\n", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.duplicate_keys_standard" + "Table": "duplicate_keys_standard" }, - "ESTUARY_SCHEMA%2Fformatted_strings": { + "formatted_strings": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO ESTUARY_SCHEMA.formatted_strings (\n\tid, date, datetime, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS date, $1[2] AS datetime, $1[3] AS flow_published_at, $1[4] AS int_and_str, $1[5] AS int_str, $1[6] AS num_and_str, $1[7] AS num_str, $1[8] AS time, $1[9] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO formatted_strings (\n\tid, date, datetime, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS date, $1[2] AS datetime, $1[3] AS flow_published_at, $1[4] AS int_and_str, $1[5] AS int_str, $1[6] AS num_and_str, $1[7] AS num_str, $1[8] AS time, $1[9] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.formatted_strings" + "Table": "formatted_strings" }, - "ESTUARY_SCHEMA%2Fmultiple_types": { + "multiple_types": { "PipeFiles": null, "PipeName": "", - "Query": "\nMERGE INTO ESTUARY_SCHEMA.multiple_types AS l\nUSING (\n\tSELECT $1[0] AS id, $1[1] AS array_int, $1[2] AS bool_field, $1[3] AS float_field, $1[4] AS flow_published_at, $1[5] AS multiple, $1[6] AS nested, $1[7] AS nullable_int, $1[8] AS str_field, $1[9] AS flow_document\n\tFROM \n) AS r\nON l.id = r.id\nWHEN MATCHED AND IS_NULL_VALUE(r.flow_document) THEN\n\tDELETE\nWHEN MATCHED THEN\n\tUPDATE SET l.array_int = r.array_int, l.bool_field = r.bool_field, l.float_field = r.float_field, l.flow_published_at = r.flow_published_at, l.multiple = r.multiple, l.nested = r.nested, l.nullable_int = r.nullable_int, l.str_field = r.str_field, l.flow_document = r.flow_document\nWHEN NOT MATCHED THEN\n\tINSERT (id, array_int, bool_field, float_field, flow_published_at, multiple, nested, nullable_int, str_field, flow_document)\n\tVALUES (r.id, r.array_int, r.bool_field, r.float_field, r.flow_published_at, r.multiple, r.nested, r.nullable_int, r.str_field, r.flow_document);\n", + "Query": "\nMERGE INTO multiple_types AS l\nUSING (\n\tSELECT $1[0] AS id, $1[1] AS array_int, $1[2] AS bool_field, $1[3] AS float_field, $1[4] AS flow_published_at, $1[5] AS multiple, $1[6] AS nested, $1[7] AS nullable_int, $1[8] AS str_field, $1[9] AS flow_document\n\tFROM \n) AS r\nON l.id = r.id\nWHEN MATCHED AND IS_NULL_VALUE(r.flow_document) THEN\n\tDELETE\nWHEN MATCHED THEN\n\tUPDATE SET l.array_int = r.array_int, l.bool_field = r.bool_field, l.float_field = r.float_field, l.flow_published_at = r.flow_published_at, l.multiple = r.multiple, l.nested = r.nested, l.nullable_int = r.nullable_int, l.str_field = r.str_field, l.flow_document = r.flow_document\nWHEN NOT MATCHED THEN\n\tINSERT (id, array_int, bool_field, float_field, flow_published_at, multiple, nested, nullable_int, str_field, flow_document)\n\tVALUES (r.id, r.array_int, r.bool_field, r.float_field, r.flow_published_at, r.multiple, r.nested, r.nullable_int, r.str_field, r.flow_document);\n", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.multiple_types" + "Table": "multiple_types" }, - "ESTUARY_SCHEMA%2Fsimple": { + "simple": { "PipeFiles": null, "PipeName": "", - "Query": "\nCOPY INTO ESTUARY_SCHEMA.simple (\n\tid, canary, flow_published_at, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS canary, $1[2] AS flow_published_at, $1[3] AS flow_document\n\tFROM \n);\n", + "Query": "\nCOPY INTO simple (\n\tid, canary, flow_published_at, flow_document\n) FROM (\n\tSELECT $1[0] AS id, $1[1] AS canary, $1[2] AS flow_published_at, $1[3] AS flow_document\n\tFROM \n);\n", "StagedDir": "", - "Table": "ESTUARY_SCHEMA.simple" + "Table": "simple" } } } @@ -362,13 +362,13 @@ "state": { "mergePatch": true, "updated": { - "ESTUARY_SCHEMA%2Fduplicate_keys_delta": null, - "ESTUARY_SCHEMA%2Fduplicate_keys_delta_exclude_flow_doc": null, - "ESTUARY_SCHEMA%2Fduplicate_keys_standard": null, - "ESTUARY_SCHEMA%2Fformatted_strings": null, - "ESTUARY_SCHEMA%2Fmultiple_types": null, - "ESTUARY_SCHEMA%2Fsimple": null, - "ESTUARY_SCHEMA%2Fsymbols": null + "duplicate_keys_delta": null, + "duplicate_keys_delta_exclude_flow_doc": null, + "duplicate_keys_standard": null, + "formatted_strings": null, + "multiple_types": null, + "simple": null, + "symbols": null } } } From a77eef46f2e54e426a454c8cd6ef511d6819f29b Mon Sep 17 00:00:00 2001 From: Will Baker Date: Thu, 14 Mar 2024 16:33:20 -0400 Subject: [PATCH 10/96] materialize-snowflake: USE SCHEMA for alter table executions Make sure we use the same connection that ran the USE SCHEMA command when running table alterations. Also make sure to return connections to the connection pool when done. --- materialize-snowflake/client.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/materialize-snowflake/client.go b/materialize-snowflake/client.go index 1e1f210de2..b1f053ce34 100644 --- a/materialize-snowflake/client.go +++ b/materialize-snowflake/client.go @@ -65,6 +65,7 @@ func (c *client) CreateTable(ctx context.Context, tc sql.TableCreate) error { if err != nil { return err } + defer conn.Close() if _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %q", schemaName)); err != nil { return err @@ -86,6 +87,8 @@ func (c *client) DeleteTable(ctx context.Context, path []string) (string, boiler if err != nil { return err } + defer conn.Close() + if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %q", c.cfg.Schema)); err != nil { return err } @@ -106,10 +109,12 @@ func (c *client) AlterTable(ctx context.Context, ta sql.TableAlter) (string, boi if err != nil { return err } + defer conn.Close() + if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %q", c.cfg.Schema)); err != nil { return err } - _, err = c.db.ExecContext(ctx, alterColumnStmt) + _, err = conn.ExecContext(ctx, alterColumnStmt) return err }, nil } From 6aa8f8e99cdb405cf7ab8b02f6ef856fdff73f7b Mon Sep 17 00:00:00 2001 From: Will Baker Date: Mon, 11 Mar 2024 17:36:49 -0400 Subject: [PATCH 11/96] materialize-bigquery: create numeric formatted string column key fields as strings columns For string fields with `format: integer` or `format: number`, we have the convention of creating these columns as their underlying "string" column type if they are collection keys. This wasn't happening in `materialize-bigquery`, which causes immediate validation errors when a table like this is created. This fixes that issue, by making `materialize-bigquery` consistently with all other SQL materializations. --- materialize-bigquery/sqlgen.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/materialize-bigquery/sqlgen.go b/materialize-bigquery/sqlgen.go index bd3358d484..d87f2ac572 100644 --- a/materialize-bigquery/sqlgen.go +++ b/materialize-bigquery/sqlgen.go @@ -67,9 +67,15 @@ var bqDialect = func() sql.Dialect { sql.STRING: sql.StringTypeMapper{ Fallback: sql.NewStaticMapper("STRING"), WithFormat: map[string]sql.TypeMapper{ - "integer": sql.NewStaticMapper("BIGNUMERIC(38,0)", sql.WithElementConverter(sql.StdStrToInt())), - // https://cloud.google.com/bigquery/docs/reference/standard-sql/conversion_functions#cast_as_floating_point - "number": sql.NewStaticMapper("FLOAT64", sql.WithElementConverter(sql.StdStrToFloat("NaN", "Infinity", "-Infinity"))), + "integer": sql.PrimaryKeyMapper{ + PrimaryKey: sql.NewStaticMapper("STRING"), + Delegate: sql.NewStaticMapper("BIGNUMERIC(38,0)", sql.WithElementConverter(sql.StdStrToInt())), + }, + "number": sql.PrimaryKeyMapper{ + PrimaryKey: sql.NewStaticMapper("STRING"), + // https://cloud.google.com/bigquery/docs/reference/standard-sql/conversion_functions#cast_as_floating_point + Delegate: sql.NewStaticMapper("FLOAT64", sql.WithElementConverter(sql.StdStrToFloat("NaN", "Infinity", "-Infinity"))), + }, "date": sql.NewStaticMapper("DATE", sql.WithElementConverter(sql.ClampDate())), "date-time": sql.NewStaticMapper("TIMESTAMP", sql.WithElementConverter(sql.ClampDatetime())), }, From f5fb9615f19194967b8dfdacde8cd6fee808f94f Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 15 Mar 2024 12:30:06 +0000 Subject: [PATCH 12/96] materialize-databricks: log when finished committing changes --- materialize-databricks/driver.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/materialize-databricks/driver.go b/materialize-databricks/driver.go index bb95b68466..8b92aaf2c8 100644 --- a/materialize-databricks/driver.go +++ b/materialize-databricks/driver.go @@ -492,6 +492,8 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error if err != nil { return nil, fmt.Errorf("creating checkpoint clearing json: %w", err) } + + log.Info("store: finished committing changes") return &pf.ConnectorState{UpdatedJson: json.RawMessage(checkpointJSON), MergePatch: true}, nil } From e9006883f2cb79c218b0e6809e0b80d360c9f4a8 Mon Sep 17 00:00:00 2001 From: Jon Wihl Date: Fri, 15 Mar 2024 09:45:52 -0400 Subject: [PATCH 13/96] non nullable primary keys --- .../source_facebook_marketing/schemas/activities.json | 8 ++++---- .../source_facebook_marketing/schemas/ad_account.json | 2 +- .../source_facebook_marketing/schemas/ad_creatives.json | 2 +- .../source_facebook_marketing/schemas/ad_sets.json | 2 +- .../source_facebook_marketing/schemas/ads.json | 2 +- .../source_facebook_marketing/schemas/ads_insights.json | 6 +++--- .../source_facebook_marketing/schemas/campaigns.json | 2 +- .../schemas/custom_conversions.json | 2 +- .../source_facebook_marketing/schemas/images.json | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/activities.json b/source-facebook-marketing/source_facebook_marketing/schemas/activities.json index 3864939e59..38c9aecded 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/activities.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/activities.json @@ -1,13 +1,13 @@ { "properties": { "actor_id": { - "type": ["null", "string"] + "type": ["string"] }, "actor_name": { "type": ["null", "string"] }, "application_id": { - "type": ["null", "string"] + "type": ["string"] }, "application_name": { "type": ["null", "string"] @@ -19,13 +19,13 @@ "type": "string" }, "event_type": { - "type": ["null", "string"] + "type": ["string"] }, "extra_data": { "type": ["null", "string"] }, "object_id": { - "type": ["null", "string"] + "type": ["string"] }, "object_name": { "type": ["null", "string"] diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json index c25c762c38..cea9a1cdc8 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json @@ -140,7 +140,7 @@ "type": ["null", "boolean"] }, "id": { - "type": ["null", "string"] + "type": ["string"] }, "io_number": { "type": ["null", "number"] diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json index 878064b6a5..27154319a5 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json @@ -10,7 +10,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "string"] + "type": ["string"] }, "account_id": { "type": ["null", "string"] diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json index ffe7fa8196..5abb863f0b 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json @@ -37,7 +37,7 @@ } }, "id": { - "type": ["null", "string"] + "type": ["string"] }, "account_id": { "type": ["null", "string"] diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads.json index ebcfabb541..985e75ab9f 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads.json @@ -71,7 +71,7 @@ } }, "id": { - "type": ["null", "string"] + "type": ["string"] }, "updated_time": { "type": ["null", "string"] diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json index 2c8435a04f..600fd6bb3a 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json @@ -4,7 +4,7 @@ "type": ["null", "string"] }, "account_id": { - "type": ["null", "string"] + "type": ["string"] }, "account_name": { "type": ["null", "string"] @@ -19,7 +19,7 @@ "$ref": "ads_action_stats.json" }, "ad_id": { - "type": ["null", "string"] + "type": ["string"] }, "ad_impression_actions": { "$ref": "ads_action_stats.json" @@ -156,7 +156,7 @@ }, "date_start": { "format": "date", - "type": ["null", "string"] + "type": ["string"] }, "date_stop": { "format": "date", diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json b/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json index 55f8d272ea..65d27ac88b 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json @@ -45,7 +45,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "string"] + "type": ["string"] }, "issues_info": { "type": ["null", "array"], diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/custom_conversions.json b/source-facebook-marketing/source_facebook_marketing/schemas/custom_conversions.json index e499717717..6b69ee39a0 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/custom_conversions.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/custom_conversions.json @@ -1,7 +1,7 @@ { "properties": { "id": { - "type": ["null", "string"] + "type": ["string"] }, "account_id": { "type": ["null", "string"] diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/images.json b/source-facebook-marketing/source_facebook_marketing/schemas/images.json index 12a502982d..af05b2296a 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/images.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/images.json @@ -6,7 +6,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "string"] + "type": ["string"] }, "name": { "type": ["null", "string"] From 1d81bc7569cd90e9b3d9f689f7687f100169d0da Mon Sep 17 00:00:00 2001 From: Jon Wihl Date: Fri, 15 Mar 2024 09:47:53 -0400 Subject: [PATCH 14/96] non nullable keys for insights breakdowns --- .../schemas/ads_insights_breakdowns.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json index cb8824fbf5..6493ccdceb 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json @@ -1,16 +1,16 @@ { "properties": { "ad_format_asset": { "type": ["null", "string"] }, - "age": { "type": ["null", "string"] }, + "age": { "type": ["string"] }, "app_id": { "type": ["null", "string"] }, "body_asset": { "type": ["null", "string"] }, "call_to_action_asset": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] }, + "country": { "type": ["string"] }, "description_asset": { "type": ["null", "string"] }, "device_platform": { "type": ["null", "string"] }, - "dma": { "type": ["null", "string"] }, + "dma": { "type": ["string"] }, "frequency_value": { "type": ["null", "string"] }, - "gender": { "type": ["null", "string"] }, + "gender": { "type": ["string"] }, "hourly_stats_aggregated_by_advertiser_time_zone": { "type": ["null", "string"] }, @@ -24,7 +24,7 @@ "platform_position": { "type": ["null", "string"] }, "product_id": { "type": ["null", "string"] }, "publisher_platform": { "type": ["null", "string"] }, - "region": { "type": ["null", "string"] }, + "region": { "type": ["string"] }, "skan_conversion_id": { "type": ["null", "string"] }, "title_asset": { "type": ["null", "string"] }, "video_asset": { "type": ["null", "string"] } From 5c033ff2cb82478746408212a07bf5e94f6c8ddd Mon Sep 17 00:00:00 2001 From: Jon Wihl Date: Fri, 15 Mar 2024 09:49:45 -0400 Subject: [PATCH 15/96] More non nullable keys --- .../schemas/ads_insights_breakdowns.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json index 6493ccdceb..5d3227f62a 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json @@ -18,12 +18,12 @@ "type": ["null", "string"] }, "image_asset": { "type": ["null", "string"] }, - "impression_device": { "type": ["null", "string"] }, + "impression_device": { "type": ["string"] }, "link_url_asset": { "type": ["null", "string"] }, "place_page_id": { "type": ["null", "string"] }, - "platform_position": { "type": ["null", "string"] }, + "platform_position": { "type": ["string"] }, "product_id": { "type": ["null", "string"] }, - "publisher_platform": { "type": ["null", "string"] }, + "publisher_platform": { "type": ["string"] }, "region": { "type": ["string"] }, "skan_conversion_id": { "type": ["null", "string"] }, "title_asset": { "type": ["null", "string"] }, From c136049559e2cfd4a5e8b01349c0e9ac02ef823b Mon Sep 17 00:00:00 2001 From: Jon Wihl Date: Fri, 15 Mar 2024 10:11:29 -0400 Subject: [PATCH 16/96] Update snapshot testing --- .../snapshots__discover__capture.stdout.json | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json index bf00321105..f1bf9029af 100644 --- a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json @@ -271,7 +271,6 @@ }, "id": { "type": [ - "null", "string" ] }, @@ -608,7 +607,6 @@ }, "id": { "type": [ - "null", "string" ] }, @@ -3333,7 +3331,6 @@ }, "id": { "type": [ - "null", "string" ] }, @@ -6541,7 +6538,6 @@ }, "id": { "type": [ - "null", "string" ] }, @@ -12209,7 +12205,6 @@ }, "account_id": { "type": [ - "null", "string" ] }, @@ -12443,7 +12438,6 @@ }, "ad_id": { "type": [ - "null", "string" ] }, @@ -14007,7 +14001,6 @@ "date_start": { "format": "date", "type": [ - "null", "string" ] }, @@ -15940,7 +15933,6 @@ }, "account_id": { "type": [ - "null", "string" ] }, @@ -16174,7 +16166,6 @@ }, "ad_id": { "type": [ - "null", "string" ] }, @@ -17738,7 +17729,6 @@ "date_start": { "format": "date", "type": [ - "null", "string" ] }, @@ -19632,13 +19622,11 @@ }, "age": { "type": [ - "null", "string" ] }, "gender": { "type": [ - "null", "string" ] }, @@ -19685,7 +19673,6 @@ }, "account_id": { "type": [ - "null", "string" ] }, @@ -19919,7 +19906,6 @@ }, "ad_id": { "type": [ - "null", "string" ] }, @@ -21483,7 +21469,6 @@ "date_start": { "format": "date", "type": [ - "null", "string" ] }, @@ -23377,7 +23362,6 @@ }, "country": { "type": [ - "null", "string" ] }, @@ -23423,7 +23407,6 @@ }, "account_id": { "type": [ - "null", "string" ] }, @@ -23657,7 +23640,6 @@ }, "ad_id": { "type": [ - "null", "string" ] }, @@ -25221,7 +25203,6 @@ "date_start": { "format": "date", "type": [ - "null", "string" ] }, @@ -27115,7 +27096,6 @@ }, "region": { "type": [ - "null", "string" ] }, @@ -27161,7 +27141,6 @@ }, "account_id": { "type": [ - "null", "string" ] }, @@ -27395,7 +27374,6 @@ }, "ad_id": { "type": [ - "null", "string" ] }, @@ -28959,7 +28937,6 @@ "date_start": { "format": "date", "type": [ - "null", "string" ] }, @@ -30853,7 +30830,6 @@ }, "dma": { "type": [ - "null", "string" ] }, @@ -30899,7 +30875,6 @@ }, "account_id": { "type": [ - "null", "string" ] }, @@ -31133,7 +31108,6 @@ }, "ad_id": { "type": [ - "null", "string" ] }, @@ -32697,7 +32671,6 @@ "date_start": { "format": "date", "type": [ - "null", "string" ] }, @@ -34591,19 +34564,16 @@ }, "publisher_platform": { "type": [ - "null", "string" ] }, "platform_position": { "type": [ - "null", "string" ] }, "impression_device": { "type": [ - "null", "string" ] }, @@ -34651,7 +34621,6 @@ }, "account_id": { "type": [ - "null", "string" ] }, @@ -34885,7 +34854,6 @@ }, "ad_id": { "type": [ - "null", "string" ] }, @@ -36449,7 +36417,6 @@ "date_start": { "format": "date", "type": [ - "null", "string" ] }, @@ -38444,7 +38411,6 @@ }, "id": { "type": [ - "null", "string" ] }, @@ -38567,7 +38533,6 @@ "properties": { "id": { "type": [ - "null", "string" ] }, @@ -38713,7 +38678,6 @@ }, "id": { "type": [ - "null", "string" ] }, @@ -39048,7 +39012,6 @@ "properties": { "actor_id": { "type": [ - "null", "string" ] }, @@ -39060,7 +39023,6 @@ }, "application_id": { "type": [ - "null", "string" ] }, @@ -39081,7 +39043,6 @@ }, "event_type": { "type": [ - "null", "string" ] }, @@ -39093,7 +39054,6 @@ }, "object_id": { "type": [ - "null", "string" ] }, @@ -39152,7 +39112,6 @@ "properties": { "account_id": { "type": [ - "null", "string" ] }, @@ -39164,7 +39123,6 @@ }, "ad_id": { "type": [ - "null", "string" ] }, @@ -39207,7 +39165,6 @@ "date_start": { "format": "date", "type": [ - "null", "string" ] }, @@ -39312,7 +39269,6 @@ }, "publisher_platform": { "type": [ - "null", "string" ] }, From ba5d039c2bae781e3271adbcb8d307c07bb59abf Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 15 Mar 2024 15:42:13 +0000 Subject: [PATCH 17/96] facebook: ads_insights primary keys must be non-nullable --- .../schemas/activities.json | 3 +- .../schemas/ad_account.json | 3 +- .../schemas/ad_creatives.json | 3 +- .../schemas/ad_sets.json | 3 +- .../schemas/ads.json | 3 +- .../schemas/ads_insights.json | 3 +- .../schemas/ads_insights_breakdowns.json | 16 ++-- .../schemas/campaigns.json | 3 +- .../schemas/images.json | 3 +- .../schemas/videos.json | 3 +- .../streams/base_insight_streams.py | 17 +++- .../snapshots__discover__capture.stdout.json | 86 +++++++++++++++++-- 12 files changed, 123 insertions(+), 23 deletions(-) diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/activities.json b/source-facebook-marketing/source_facebook_marketing/schemas/activities.json index 38c9aecded..508d273e5e 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/activities.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/activities.json @@ -37,5 +37,6 @@ "type": ["null", "string"] } }, - "type": ["object"] + "type": ["object"], + "required": ["actor_id", "event_time", "event_type", "object_id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json index cea9a1cdc8..9da6a9b4c7 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json @@ -263,5 +263,6 @@ } } } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json index 27154319a5..fb944e431e 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json @@ -1375,5 +1375,6 @@ "type": ["null", "string"] } }, - "type": ["object"] + "type": ["object"], + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json index 5abb863f0b..1224293256 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json @@ -110,5 +110,6 @@ } } } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads.json index 985e75ab9f..247068d631 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads.json @@ -446,5 +446,6 @@ } } } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json index 600fd6bb3a..42c19161c3 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json @@ -338,5 +338,6 @@ "type": ["null", "number"] } }, - "type": ["object"] + "type": ["object"], + "required": ["account_id", "ad_id", "date_start"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json index 5d3227f62a..cb8824fbf5 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json @@ -1,16 +1,16 @@ { "properties": { "ad_format_asset": { "type": ["null", "string"] }, - "age": { "type": ["string"] }, + "age": { "type": ["null", "string"] }, "app_id": { "type": ["null", "string"] }, "body_asset": { "type": ["null", "string"] }, "call_to_action_asset": { "type": ["null", "string"] }, - "country": { "type": ["string"] }, + "country": { "type": ["null", "string"] }, "description_asset": { "type": ["null", "string"] }, "device_platform": { "type": ["null", "string"] }, - "dma": { "type": ["string"] }, + "dma": { "type": ["null", "string"] }, "frequency_value": { "type": ["null", "string"] }, - "gender": { "type": ["string"] }, + "gender": { "type": ["null", "string"] }, "hourly_stats_aggregated_by_advertiser_time_zone": { "type": ["null", "string"] }, @@ -18,13 +18,13 @@ "type": ["null", "string"] }, "image_asset": { "type": ["null", "string"] }, - "impression_device": { "type": ["string"] }, + "impression_device": { "type": ["null", "string"] }, "link_url_asset": { "type": ["null", "string"] }, "place_page_id": { "type": ["null", "string"] }, - "platform_position": { "type": ["string"] }, + "platform_position": { "type": ["null", "string"] }, "product_id": { "type": ["null", "string"] }, - "publisher_platform": { "type": ["string"] }, - "region": { "type": ["string"] }, + "publisher_platform": { "type": ["null", "string"] }, + "region": { "type": ["null", "string"] }, "skan_conversion_id": { "type": ["null", "string"] }, "title_asset": { "type": ["null", "string"] }, "video_asset": { "type": ["null", "string"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json b/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json index 65d27ac88b..a5ec954b7c 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json @@ -107,5 +107,6 @@ "type": "string" } }, - "type": ["object"] + "type": ["object"], + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/images.json b/source-facebook-marketing/source_facebook_marketing/schemas/images.json index af05b2296a..1d34c5cdbc 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/images.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/images.json @@ -56,5 +56,6 @@ "width": { "type": ["null", "integer"] } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/videos.json b/source-facebook-marketing/source_facebook_marketing/schemas/videos.json index 3239f2534b..52e2dac43b 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/videos.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/videos.json @@ -107,5 +107,6 @@ "views": { "type": ["null", "integer"] } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 18f180b8cf..6c248a85e6 100644 --- a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -220,7 +220,6 @@ def check_breakdowns(self): def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - """Slice by date periods and schedule async job for each period, run at most MAX_ASYNC_JOBS jobs at the same time. This solution for Async was chosen because: 1. we should commit state after each successful job @@ -297,6 +296,22 @@ def get_json_schema(self) -> Mapping[str, Any]: if self.breakdowns: breakdowns_properties = loader.get_schema("ads_insights_breakdowns")["properties"] schema["properties"].update({prop: breakdowns_properties[prop] for prop in self.breakdowns}) + + for k in self.primary_key: + if k not in schema["properties"]: + continue + + tys = schema["properties"][k]["type"] + + if "null" in tys: + schema["properties"][k]["type"].remove("null") + + if "required" not in schema: + schema["required"] = [] + + if k not in schema["required"]: + schema["required"].append(k) + return schema @cached_property diff --git a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json index f1bf9029af..050e256c63 100644 --- a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json @@ -517,7 +517,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -3192,7 +3195,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -6504,7 +6510,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -12180,6 +12189,9 @@ }, "type": [ "object" + ], + "required": [ + "id" ] }, "key": [ @@ -15906,6 +15918,11 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start" ] }, "key": [ @@ -19644,6 +19661,13 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "age", + "gender" ] }, "key": [ @@ -23379,6 +23403,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "country" ] }, "key": [ @@ -27113,6 +27143,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "region" ] }, "key": [ @@ -30847,6 +30883,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "dma" ] }, "key": [ @@ -34591,6 +34633,14 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "publisher_platform", + "platform_position", + "impression_device" ] }, "key": [ @@ -38322,6 +38372,11 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start" ] }, "key": [ @@ -38517,6 +38572,9 @@ }, "type": [ "object" + ], + "required": [ + "id" ] }, "key": [ @@ -38782,7 +38840,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -38993,7 +39054,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -39089,6 +39153,12 @@ }, "type": [ "object" + ], + "required": [ + "actor_id", + "event_time", + "event_type", + "object_id" ] }, "key": [ @@ -39286,6 +39356,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "publisher_platform" ] }, "key": [ From 4065367ea6cb9a4d9862a980a90b51c8e7987781 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 15 Mar 2024 17:40:11 +0000 Subject: [PATCH 18/96] materialize-snowflake: use dialect identifier transformer for schema --- materialize-snowflake/client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/materialize-snowflake/client.go b/materialize-snowflake/client.go index b1f053ce34..ef4c1aad4a 100644 --- a/materialize-snowflake/client.go +++ b/materialize-snowflake/client.go @@ -67,11 +67,11 @@ func (c *client) CreateTable(ctx context.Context, tc sql.TableCreate) error { } defer conn.Close() - if _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %q", schemaName)); err != nil { + if _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", c.ep.Dialect.Identifier(schemaName))); err != nil { return err } - if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %q", c.cfg.Schema)); err != nil { + if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %s", c.ep.Dialect.Identifier(c.cfg.Schema))); err != nil { return err } @@ -89,7 +89,7 @@ func (c *client) DeleteTable(ctx context.Context, path []string) (string, boiler } defer conn.Close() - if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %q", c.cfg.Schema)); err != nil { + if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %s", c.ep.Dialect.Identifier(c.cfg.Schema))); err != nil { return err } _, err = conn.ExecContext(ctx, stmt) @@ -111,7 +111,7 @@ func (c *client) AlterTable(ctx context.Context, ta sql.TableAlter) (string, boi } defer conn.Close() - if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %q", c.cfg.Schema)); err != nil { + if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE SCHEMA %s", c.ep.Dialect.Identifier(c.cfg.Schema))); err != nil { return err } _, err = conn.ExecContext(ctx, alterColumnStmt) From d59e97e99a8cf2dd0ec0251e8adff260ab8c5337 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Thu, 14 Mar 2024 14:30:35 -0500 Subject: [PATCH 19/96] source-mysql: Omit unexpected `DB_ROW_HASH_1` column values See https://github.com/estuary/connectors/issues/1344 for the full explanation, but this commit adds a special-case to our document value translation logic to exclude columns by the name of `DB_ROW_HASH_1` for which we have no type information. There is no accompanying test because this can only be reproduced on MariaDB and we use MySQL for our default test setup, but I have tested this locally and confirmed that it fixes what would otherwise be a capture error. --- source-mysql/discovery.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source-mysql/discovery.go b/source-mysql/discovery.go index 2a4c051c01..8fd678d07d 100644 --- a/source-mysql/discovery.go +++ b/source-mysql/discovery.go @@ -171,6 +171,15 @@ func (db *mysqlDatabase) translateRecordFields(columnTypes map[string]interface{ return nil } for id, val := range f { + // MariaDB versions 10.4 and up include a synthetic `DB_ROW_HASH_1` column in the + // binlog row change events for certain tables with unique hash indices (see issue + // https://github.com/estuary/connectors/issues/1344). In such cases we won't have + // any type information for the column, but we also don't want it anyway, so as a + // special case we just delete the 'DB_ROW_HASH_1' property from the document. + if id == "DB_ROW_HASH_1" && columnTypes[id] == nil { + delete(f, id) + continue + } var translated, err = db.translateRecordField(columnTypes[id], val) if err != nil { return fmt.Errorf("error translating field %q value %v: %w", id, val, err) From 25f76d52b92120f23c0374ff88a14c38af185593 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Fri, 15 Mar 2024 13:12:41 -0500 Subject: [PATCH 20/96] source-{postgres,redshift}-batch: Use 'pg_catalog' for discovery This commit replaces the table and primary key discovery queries used by the batch Postgres and Redshift captures. Previously these queries read from various `information_schema` views, but in practice getting permission grants set up so those views return the needed information could be quite difficult. The fix is to drop down a level and consult the 'pg_catalog' tables directly. This ended up being a bit more complicated than expected, because Redshift is based on a really old version of Postgres and also changed some things around, but I think the new queries that I ended up with should work well enough for now. --- .../.snapshots/TestBasicCapture-Discovery | 9 +- .../.snapshots/TestBasicDatatypes-Discovery | 9 +- .../.snapshots/TestKeyDiscovery | 77 +++++++++++++++++ .../.snapshots/TestSchemaFilter-FilteredIn | 9 +- .../.snapshots/TestSchemaFilter-Unfiltered | 9 +- source-postgres-batch/driver.go | 80 +++++++++--------- source-postgres-batch/main_test.go | 14 ++++ .../.snapshots/TestKeyDiscovery | 75 +++++++++++++++++ source-redshift-batch/driver.go | 84 ++++++++++--------- source-redshift-batch/main_test.go | 14 ++++ 10 files changed, 289 insertions(+), 91 deletions(-) create mode 100644 source-postgres-batch/.snapshots/TestKeyDiscovery create mode 100644 source-redshift-batch/.snapshots/TestKeyDiscovery diff --git a/source-postgres-batch/.snapshots/TestBasicCapture-Discovery b/source-postgres-batch/.snapshots/TestBasicCapture-Discovery index cba977e33e..643fb1c3de 100644 --- a/source-postgres-batch/.snapshots/TestBasicCapture-Discovery +++ b/source-postgres-batch/.snapshots/TestBasicCapture-Discovery @@ -15,7 +15,8 @@ Binding 0: "read_schema_json": { "type": "object", "required": [ - "_meta" + "_meta", + "id" ], "properties": { "_meta": { @@ -39,13 +40,15 @@ Binding 0: "polled", "index" ] + }, + "id": { + "type": "integer" } }, "x-infer-schema": true }, "key": [ - "/_meta/polled", - "/_meta/index" + "/id" ], "projections": null }, diff --git a/source-postgres-batch/.snapshots/TestBasicDatatypes-Discovery b/source-postgres-batch/.snapshots/TestBasicDatatypes-Discovery index 6c1ba723fa..38594a637b 100644 --- a/source-postgres-batch/.snapshots/TestBasicDatatypes-Discovery +++ b/source-postgres-batch/.snapshots/TestBasicDatatypes-Discovery @@ -15,7 +15,8 @@ Binding 0: "read_schema_json": { "type": "object", "required": [ - "_meta" + "_meta", + "id" ], "properties": { "_meta": { @@ -39,13 +40,15 @@ Binding 0: "polled", "index" ] + }, + "id": { + "type": "integer" } }, "x-infer-schema": true }, "key": [ - "/_meta/polled", - "/_meta/index" + "/id" ], "projections": null }, diff --git a/source-postgres-batch/.snapshots/TestKeyDiscovery b/source-postgres-batch/.snapshots/TestKeyDiscovery new file mode 100644 index 0000000000..b0d977757d --- /dev/null +++ b/source-postgres-batch/.snapshots/TestKeyDiscovery @@ -0,0 +1,77 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_key_discovery_329932", + "template": "{{if .IsFirstQuery -}}\n SELECT xmin AS txid, * FROM \"test\".\"key_discovery_329932\" ORDER BY xmin::text::bigint;\n{{- else -}}\n SELECT xmin AS txid, * FROM \"test\".\"key_discovery_329932\" WHERE xmin::text::bigint \u003e $1 ORDER BY xmin::text::bigint;\n{{- end}}", + "cursor": [ + "txid" + ] + }, + "resource_path": [ + "test_key_discovery_329932" + ], + "collection": { + "name": "acmeCo/test/test_key_discovery_329932", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "k_smallint", + "k_int", + "k_bigint", + "k_bool", + "k_str" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-postgres-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "k_bigint": { + "type": "integer" + }, + "k_bool": { + "type": "boolean" + }, + "k_int": { + "type": "integer" + }, + "k_smallint": { + "type": "integer" + }, + "k_str": { + "type": "string" + } + }, + "x-infer-schema": true + }, + "key": [ + "/k_smallint", + "/k_int", + "/k_bigint", + "/k_bool", + "/k_str" + ], + "projections": null + }, + "state_key": "test_key_discovery_329932" + } + diff --git a/source-postgres-batch/.snapshots/TestSchemaFilter-FilteredIn b/source-postgres-batch/.snapshots/TestSchemaFilter-FilteredIn index 8afeeaaff2..e4493712cd 100644 --- a/source-postgres-batch/.snapshots/TestSchemaFilter-FilteredIn +++ b/source-postgres-batch/.snapshots/TestSchemaFilter-FilteredIn @@ -15,7 +15,8 @@ Binding 0: "read_schema_json": { "type": "object", "required": [ - "_meta" + "_meta", + "id" ], "properties": { "_meta": { @@ -39,13 +40,15 @@ Binding 0: "polled", "index" ] + }, + "id": { + "type": "integer" } }, "x-infer-schema": true }, "key": [ - "/_meta/polled", - "/_meta/index" + "/id" ], "projections": null }, diff --git a/source-postgres-batch/.snapshots/TestSchemaFilter-Unfiltered b/source-postgres-batch/.snapshots/TestSchemaFilter-Unfiltered index 8afeeaaff2..e4493712cd 100644 --- a/source-postgres-batch/.snapshots/TestSchemaFilter-Unfiltered +++ b/source-postgres-batch/.snapshots/TestSchemaFilter-Unfiltered @@ -15,7 +15,8 @@ Binding 0: "read_schema_json": { "type": "object", "required": [ - "_meta" + "_meta", + "id" ], "properties": { "_meta": { @@ -39,13 +40,15 @@ Binding 0: "polled", "index" ] + }, + "id": { + "type": "integer" } }, "x-infer-schema": true }, "key": [ - "/_meta/polled", - "/_meta/index" + "/id" ], "projections": null }, diff --git a/source-postgres-batch/driver.go b/source-postgres-batch/driver.go index ed72a8b8fd..403230d226 100644 --- a/source-postgres-batch/driver.go +++ b/source-postgres-batch/driver.go @@ -155,14 +155,6 @@ func (BatchSQLDriver) Apply(ctx context.Context, req *pc.Request_Apply) (*pc.Res return &pc.Response_Applied{ActionDescription: ""}, nil } -var excludedSystemSchemas = []string{ - "pg_catalog", - "information_schema", - "pg_internal", - "catalog_history", - "cron", -} - // Discover enumerates tables and views from `information_schema.tables` and generates // placeholder capture queries for thos tables. func (drv *BatchSQLDriver) Discover(ctx context.Context, req *pc.Request_Discover) (*pc.Response_Discovered, error) { @@ -198,11 +190,6 @@ func (drv *BatchSQLDriver) Discover(ctx context.Context, req *pc.Request_Discove var bindings []*pc.Response_Discovered_Binding for _, table := range tables { - // Exclude tables in "system schemas" such as information_schema or pg_catalog. - if slices.Contains(excludedSystemSchemas, table.Schema) { - continue - } - var tableID = table.Schema + "." + table.Name var recommendedName = recommendedCatalogName(table.Schema, table.Name) @@ -267,12 +254,25 @@ type discoveredTable struct { func discoverTables(ctx context.Context, db *sql.DB, discoverSchemas []string) ([]*discoveredTable, error) { var query = new(strings.Builder) var args []any - fmt.Fprintf(query, "SELECT table_schema, table_name, table_type FROM information_schema.tables") + + fmt.Fprintf(query, "SELECT n.nspname AS table_schema,") + fmt.Fprintf(query, " c.relname AS table_name,") + fmt.Fprintf(query, " CASE") + fmt.Fprintf(query, " WHEN n.oid = pg_my_temp_schema() THEN 'LOCAL TEMPORARY'::text") + fmt.Fprintf(query, " WHEN c.relkind = ANY (ARRAY['r'::\"char\", 'p'::\"char\"]) THEN 'BASE TABLE'::text") + fmt.Fprintf(query, " WHEN c.relkind = 'v'::\"char\" THEN 'VIEW'::text") + fmt.Fprintf(query, " WHEN c.relkind = 'f'::\"char\" THEN 'FOREIGN'::text") + fmt.Fprintf(query, " ELSE ''::text") + fmt.Fprintf(query, " END::information_schema.character_data AS table_type") + fmt.Fprintf(query, " FROM pg_catalog.pg_class c") + fmt.Fprintf(query, " JOIN pg_catalog.pg_namespace n ON (n.oid = c.relnamespace)") + fmt.Fprintf(query, " WHERE n.nspname NOT IN ('pg_catalog', 'pg_internal', 'information_schema', 'catalog_history', 'cron')") + fmt.Fprintf(query, " AND c.relkind IN ('r', 'p', 'v', 'f')") if len(discoverSchemas) > 0 { - fmt.Fprintf(query, " WHERE table_schema = ANY ($1)") + fmt.Fprintf(query, " AND n.nspname = ANY ($1)") args = append(args, discoverSchemas) } - fmt.Fprintf(query, ";") + fmt.Fprintf(query, " AND NOT c.relispartition;") // Exclude subpartitions of a partitioned table from discovery rows, err := db.QueryContext(ctx, query.String(), args...) if err != nil { @@ -305,29 +305,28 @@ type discoveredPrimaryKey struct { func discoverPrimaryKeys(ctx context.Context, db *sql.DB, discoverSchemas []string) ([]*discoveredPrimaryKey, error) { var query = new(strings.Builder) var args []any - // Joining on the 6-tuple {CONSTRAINT,TABLE}_{CATALOG,SCHEMA,NAME} is probably - // overkill but shouldn't hurt, and helps to make absolutely sure that we're - // matching up the constraint type with the column names/positions correctly. - fmt.Fprintf(query, "SELECT kcu.table_schema, kcu.table_name, kcu.column_name, kcu.ordinal_position, col.data_type") - fmt.Fprintf(query, " FROM information_schema.key_column_usage kcu") - fmt.Fprintf(query, " JOIN information_schema.table_constraints tcs") - fmt.Fprintf(query, " ON tcs.constraint_catalog = kcu.constraint_catalog") - fmt.Fprintf(query, " AND tcs.constraint_schema = kcu.constraint_schema") - fmt.Fprintf(query, " AND tcs.constraint_name = kcu.constraint_name") - fmt.Fprintf(query, " AND tcs.table_catalog = kcu.table_catalog") - fmt.Fprintf(query, " AND tcs.table_schema = kcu.table_schema") - fmt.Fprintf(query, " AND tcs.table_name = kcu.table_name") - fmt.Fprintf(query, " JOIN information_schema.columns col") - fmt.Fprintf(query, " ON col.table_catalog = kcu.table_catalog") - fmt.Fprintf(query, " AND col.table_schema = kcu.table_schema") - fmt.Fprintf(query, " AND col.table_name = kcu.table_name") - fmt.Fprintf(query, " AND col.column_name = kcu.column_name") - fmt.Fprintf(query, " WHERE tcs.constraint_type = 'PRIMARY KEY'") + + fmt.Fprintf(query, "SELECT result.TABLE_SCHEM, result.TABLE_NAME, result.COLUMN_NAME, result.KEY_SEQ, result.TYPE_NAME") + fmt.Fprintf(query, " FROM (") + fmt.Fprintf(query, " SELECT n.nspname AS TABLE_SCHEM,") + fmt.Fprintf(query, " ct.relname AS TABLE_NAME, a.attname AS COLUMN_NAME,") + fmt.Fprintf(query, " (information_schema._pg_expandarray(i.indkey)).n AS KEY_SEQ, ci.relname AS PK_NAME,") + fmt.Fprintf(query, " information_schema._pg_expandarray(i.indkey) AS KEYS, a.attnum AS A_ATTNUM,") + fmt.Fprintf(query, " t.typname AS TYPE_NAME") + fmt.Fprintf(query, " FROM pg_catalog.pg_class ct") + fmt.Fprintf(query, " JOIN pg_catalog.pg_attribute a ON (ct.oid = a.attrelid)") + fmt.Fprintf(query, " JOIN pg_catalog.pg_namespace n ON (ct.relnamespace = n.oid)") + fmt.Fprintf(query, " JOIN pg_catalog.pg_index i ON (a.attrelid = i.indrelid)") + fmt.Fprintf(query, " JOIN pg_catalog.pg_class ci ON (ci.oid = i.indexrelid)") + fmt.Fprintf(query, " JOIN pg_catalog.pg_type t ON (a.atttypid = t.oid)") + fmt.Fprintf(query, " WHERE i.indisprimary") if len(discoverSchemas) > 0 { - fmt.Fprintf(query, " AND kcu.table_schema = ANY ($1)") + fmt.Fprintf(query, " AND n.nspname = ANY ($1)") args = append(args, discoverSchemas) } - fmt.Fprintf(query, " ORDER BY kcu.table_schema, kcu.table_name, kcu.ordinal_position;") + fmt.Fprintf(query, " ) result") + fmt.Fprintf(query, " WHERE result.A_ATTNUM = (result.KEYS).x") + fmt.Fprintf(query, " ORDER BY result.table_name, result.pk_name, result.key_seq;") rows, err := db.QueryContext(ctx, query.String(), args...) if err != nil { @@ -374,10 +373,11 @@ func discoverPrimaryKeys(ctx context.Context, db *sql.DB, discoverSchemas []stri } var databaseTypeToJSON = map[string]*jsonschema.Schema{ - "integer": {Type: "integer"}, - "bigint": {Type: "integer"}, - "smallint": {Type: "integer"}, - "boolean": {Type: "boolean"}, + "int2": {Type: "integer"}, + "int4": {Type: "integer"}, + "int8": {Type: "integer"}, + "bool": {Type: "boolean"}, + "varchar": {Type: "string"}, } var catalogNameSanitizerRe = regexp.MustCompile(`(?i)[^a-z0-9\-_.]`) diff --git a/source-postgres-batch/main_test.go b/source-postgres-batch/main_test.go index 940198da91..378a33cce0 100644 --- a/source-postgres-batch/main_test.go +++ b/source-postgres-batch/main_test.go @@ -139,6 +139,20 @@ func TestSchemaFilter(t *testing.T) { }) } +func TestKeyDiscovery(t *testing.T) { + var ctx, cs = context.Background(), testCaptureSpec(t) + var control = testControlClient(ctx, t) + var uniqueID = "329932" + var tableName = fmt.Sprintf("test.key_discovery_%s", uniqueID) + + executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) + t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) + executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(k_smallint SMALLINT, k_int INTEGER, k_bigint BIGINT, k_bool BOOLEAN, k_str VARCHAR(8), data TEXT, PRIMARY KEY (k_smallint, k_int, k_bigint, k_bool, k_str))", tableName)) + + cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{"test"} + snapshotBindings(t, discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID))) +} + func testControlClient(ctx context.Context, t testing.TB) *sql.DB { t.Helper() if os.Getenv("TEST_DATABASE") != "yes" { diff --git a/source-redshift-batch/.snapshots/TestKeyDiscovery b/source-redshift-batch/.snapshots/TestKeyDiscovery new file mode 100644 index 0000000000..66d5f46c61 --- /dev/null +++ b/source-redshift-batch/.snapshots/TestKeyDiscovery @@ -0,0 +1,75 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_key_discovery_329932", + "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"key_discovery_329932\"\n {{- else -}}\n SELECT * FROM \"test\".\"key_discovery_329932\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"key_discovery_329932\";\n{{- end}}\n", + "cursor": null + }, + "resource_path": [ + "test_key_discovery_329932" + ], + "collection": { + "name": "acmeCo/test/test_key_discovery_329932", + "read_schema_json": { + "type": "object", + "required": [ + "_meta", + "k_smallint", + "k_int", + "k_bigint", + "k_bool", + "k_str" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-redshift-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "k_bigint": { + "type": "integer" + }, + "k_bool": { + "type": "boolean" + }, + "k_int": { + "type": "integer" + }, + "k_smallint": { + "type": "integer" + }, + "k_str": { + "type": "string" + } + }, + "x-infer-schema": true + }, + "key": [ + "/k_smallint", + "/k_int", + "/k_bigint", + "/k_bool", + "/k_str" + ], + "projections": null + }, + "state_key": "test_key_discovery_329932" + } + diff --git a/source-redshift-batch/driver.go b/source-redshift-batch/driver.go index 6a0b34eb3c..1ae403d7e9 100644 --- a/source-redshift-batch/driver.go +++ b/source-redshift-batch/driver.go @@ -159,14 +159,6 @@ func (BatchSQLDriver) Apply(ctx context.Context, req *pc.Request_Apply) (*pc.Res return &pc.Response_Applied{ActionDescription: ""}, nil } -var excludedSystemSchemas = []string{ - "pg_catalog", - "information_schema", - "pg_internal", - "catalog_history", - "cron", -} - // Discover enumerates tables and views from `information_schema.tables` and generates // placeholder capture queries for thos tables. func (drv *BatchSQLDriver) Discover(ctx context.Context, req *pc.Request_Discover) (*pc.Response_Discovered, error) { @@ -202,11 +194,6 @@ func (drv *BatchSQLDriver) Discover(ctx context.Context, req *pc.Request_Discove var bindings []*pc.Response_Discovered_Binding for _, table := range tables { - // Exclude tables in "system schemas" such as information_schema or pg_catalog. - if slices.Contains(excludedSystemSchemas, table.Schema) { - continue - } - var tableID = table.Schema + "." + table.Name var recommendedName = recommendedCatalogName(table.Schema, table.Name) @@ -271,9 +258,21 @@ type discoveredTable struct { func discoverTables(ctx context.Context, db *sql.DB, discoverSchemas []string) ([]*discoveredTable, error) { var query = new(strings.Builder) var args []any - fmt.Fprintf(query, "SELECT table_schema, table_name, table_type FROM information_schema.tables") + + fmt.Fprintf(query, "SELECT n.nspname AS table_schema,") + fmt.Fprintf(query, " c.relname AS table_name,") + fmt.Fprintf(query, " CASE") + fmt.Fprintf(query, " WHEN c.relkind = ANY (ARRAY['r'::\"char\", 'p'::\"char\"]) THEN 'BASE TABLE'::text") + fmt.Fprintf(query, " WHEN c.relkind = 'v'::\"char\" THEN 'VIEW'::text") + fmt.Fprintf(query, " WHEN c.relkind = 'f'::\"char\" THEN 'FOREIGN'::text") + fmt.Fprintf(query, " ELSE ''::text") + fmt.Fprintf(query, " END::information_schema.character_data AS table_type") + fmt.Fprintf(query, " FROM pg_catalog.pg_class c") + fmt.Fprintf(query, " JOIN pg_catalog.pg_namespace n ON (n.oid = c.relnamespace)") + fmt.Fprintf(query, " WHERE n.nspname NOT IN ('pg_catalog', 'pg_internal', 'information_schema', 'catalog_history', 'cron')") + fmt.Fprintf(query, " AND c.relkind IN ('r', 'p', 'v', 'f')") if len(discoverSchemas) > 0 { - fmt.Fprintf(query, " WHERE table_schema = ANY ($1)") + fmt.Fprintf(query, " AND n.nspname = ANY ($1)") args = append(args, discoverSchemas) } fmt.Fprintf(query, ";") @@ -309,29 +308,35 @@ type discoveredPrimaryKey struct { func discoverPrimaryKeys(ctx context.Context, db *sql.DB, discoverSchemas []string) ([]*discoveredPrimaryKey, error) { var query = new(strings.Builder) var args []any - // Joining on the 6-tuple {CONSTRAINT,TABLE}_{CATALOG,SCHEMA,NAME} is probably - // overkill but shouldn't hurt, and helps to make absolutely sure that we're - // matching up the constraint type with the column names/positions correctly. - fmt.Fprintf(query, "SELECT kcu.table_schema, kcu.table_name, kcu.column_name, kcu.ordinal_position, col.data_type") - fmt.Fprintf(query, " FROM information_schema.key_column_usage kcu") - fmt.Fprintf(query, " JOIN information_schema.table_constraints tcs") - fmt.Fprintf(query, " ON tcs.constraint_catalog = kcu.constraint_catalog") - fmt.Fprintf(query, " AND tcs.constraint_schema = kcu.constraint_schema") - fmt.Fprintf(query, " AND tcs.constraint_name = kcu.constraint_name") - fmt.Fprintf(query, " AND tcs.table_catalog = kcu.table_catalog") - fmt.Fprintf(query, " AND tcs.table_schema = kcu.table_schema") - fmt.Fprintf(query, " AND tcs.table_name = kcu.table_name") - fmt.Fprintf(query, " JOIN information_schema.columns col") - fmt.Fprintf(query, " ON col.table_catalog = kcu.table_catalog") - fmt.Fprintf(query, " AND col.table_schema = kcu.table_schema") - fmt.Fprintf(query, " AND col.table_name = kcu.table_name") - fmt.Fprintf(query, " AND col.column_name = kcu.column_name") - fmt.Fprintf(query, " WHERE tcs.constraint_type = 'PRIMARY KEY'") + + fmt.Fprintf(query, "SELECT nr.nspname::information_schema.sql_identifier AS table_schema,") + fmt.Fprintf(query, " r.relname::information_schema.sql_identifier AS table_name,") + fmt.Fprintf(query, " a.attname::information_schema.sql_identifier AS column_name,") + fmt.Fprintf(query, " pos.n::information_schema.cardinal_number AS ordinal_position,") + // The handling of type names here is a bit weak, refer to how the `information_schema.columns` + // view computes its `data_type` column for a more comprehensive approach. But in practice we + // don't need all of that complexity just to identify basic integer columns, so this ought to + // be sufficient for now. + fmt.Fprintf(query, " t.typname AS type_name") + fmt.Fprintf(query, " FROM pg_namespace nr,") + fmt.Fprintf(query, " pg_class r,") + fmt.Fprintf(query, " pg_attribute a,") + fmt.Fprintf(query, " pg_type t,") + fmt.Fprintf(query, " pg_constraint c,") + fmt.Fprintf(query, " generate_series(1,100,1) pos(n)") + fmt.Fprintf(query, " WHERE nr.oid = r.relnamespace") + fmt.Fprintf(query, " AND r.oid = a.attrelid") + fmt.Fprintf(query, " AND r.oid = c.conrelid") + fmt.Fprintf(query, " AND t.oid = a.atttypid") + fmt.Fprintf(query, " AND c.conkey[pos.n] = a.attnum") + fmt.Fprintf(query, " AND NOT a.attisdropped") + fmt.Fprintf(query, " AND c.contype = 'p'::\"char\"") + fmt.Fprintf(query, " AND r.relkind = 'r'::\"char\"") if len(discoverSchemas) > 0 { - fmt.Fprintf(query, " AND kcu.table_schema = ANY ($1)") + fmt.Fprintf(query, " AND nr.nspname = ANY ($1)") args = append(args, discoverSchemas) } - fmt.Fprintf(query, " ORDER BY kcu.table_schema, kcu.table_name, kcu.ordinal_position;") + fmt.Fprintf(query, " ORDER BY r.relname, pos.n;") rows, err := db.QueryContext(ctx, query.String(), args...) if err != nil { @@ -378,10 +383,11 @@ func discoverPrimaryKeys(ctx context.Context, db *sql.DB, discoverSchemas []stri } var databaseTypeToJSON = map[string]*jsonschema.Schema{ - "integer": {Type: "integer"}, - "bigint": {Type: "integer"}, - "smallint": {Type: "integer"}, - "boolean": {Type: "boolean"}, + "int2": {Type: "integer"}, + "int4": {Type: "integer"}, + "int8": {Type: "integer"}, + "bool": {Type: "boolean"}, + "varchar": {Type: "string"}, } var catalogNameSanitizerRe = regexp.MustCompile(`(?i)[^a-z0-9\-_.]`) diff --git a/source-redshift-batch/main_test.go b/source-redshift-batch/main_test.go index b97d477279..c2c66396e0 100644 --- a/source-redshift-batch/main_test.go +++ b/source-redshift-batch/main_test.go @@ -180,6 +180,20 @@ func TestSchemaFilter(t *testing.T) { }) } +func TestKeyDiscovery(t *testing.T) { + var ctx, cs = context.Background(), testCaptureSpec(t) + var control = testControlClient(ctx, t) + var uniqueID = "329932" + var tableName = fmt.Sprintf("test.key_discovery_%s", uniqueID) + + executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) + t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) + executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(k_smallint SMALLINT, k_int INTEGER, k_bigint BIGINT, k_bool BOOLEAN, k_str VARCHAR(8), data TEXT, PRIMARY KEY (k_smallint, k_int, k_bigint, k_bool, k_str))", tableName)) + + cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{"test"} + snapshotBindings(t, discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID))) +} + func testControlClient(ctx context.Context, t testing.TB) *sql.DB { t.Helper() if os.Getenv("TEST_DATABASE") != "yes" { From a6be846514537dc84082edfa8a70ddc8da495ad5 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Fri, 15 Mar 2024 15:21:27 -0500 Subject: [PATCH 21/96] source-mysql: Omit `BINARY` qualifier when doing unfiltered backfill Currently `source-mysql` has no logic to automatically select the unfiltered backfill mode, so we basically always do either a keyless or a precise keyed backfill. (This is because the whole precise/unfiltered distinction was added to deal with SQL Server text collation behavior and has not yet been fully extended to other situations where it might apply). But in order to make precise backfills work, the MySQL backfill queries slap a `BINARY` qualifier on any textual primary key columns, and this basically always forces a full-table sort which can be prohibitively expensive in some situations. It's possible to explicitly set the backfill mode for a binding to `"Normal"` in the spec, and it would be nice if doing this resulted in dropping the `BINARY` qualifier since we don't need the key ordering invariants this produces in the unfiltered mode. So that's what this commit does -- if you explicitly select the `"Normal"` backfill mode we end up performing an unfiltered backfill and the query generation logic just omits the `BINARY` when the backfill mode isn't precise. --- source-mysql/.snapshots/TestBackfillModes | 2011 +++++++++++++++++++++ source-mysql/backfill.go | 11 +- source-mysql/capture_test.go | 33 + 3 files changed, 2051 insertions(+), 4 deletions(-) create mode 100644 source-mysql/.snapshots/TestBackfillModes diff --git a/source-mysql/.snapshots/TestBackfillModes b/source-mysql/.snapshots/TestBackfillModes new file mode 100644 index 0000000000..2ad503f166 --- /dev/null +++ b/source-mysql/.snapshots/TestBackfillModes @@ -0,0 +1,2011 @@ +# ================================ +# Collection "acmeCo/test/test_backfillmodes_11837744": 1000 Documents +# ================================ +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:0"}},"data":"Data for row 0","id":"Row 0"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:1"}},"data":"Data for row 1","id":"Row 1"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:2"}},"data":"Data for row 10","id":"Row 10"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:3"}},"data":"Data for row 100","id":"Row 100"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:4"}},"data":"Data for row 101","id":"Row 101"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:5"}},"data":"Data for row 102","id":"Row 102"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:6"}},"data":"Data for row 103","id":"Row 103"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:7"}},"data":"Data for row 104","id":"Row 104"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:8"}},"data":"Data for row 105","id":"Row 105"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:9"}},"data":"Data for row 106","id":"Row 106"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:10"}},"data":"Data for row 107","id":"Row 107"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:11"}},"data":"Data for row 108","id":"Row 108"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:12"}},"data":"Data for row 109","id":"Row 109"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:13"}},"data":"Data for row 11","id":"Row 11"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:14"}},"data":"Data for row 110","id":"Row 110"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:15"}},"data":"Data for row 111","id":"Row 111"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:16"}},"data":"Data for row 112","id":"Row 112"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:17"}},"data":"Data for row 113","id":"Row 113"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:18"}},"data":"Data for row 114","id":"Row 114"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:19"}},"data":"Data for row 115","id":"Row 115"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:20"}},"data":"Data for row 116","id":"Row 116"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:21"}},"data":"Data for row 117","id":"Row 117"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:22"}},"data":"Data for row 118","id":"Row 118"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:23"}},"data":"Data for row 119","id":"Row 119"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:24"}},"data":"Data for row 12","id":"Row 12"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:25"}},"data":"Data for row 120","id":"Row 120"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:26"}},"data":"Data for row 121","id":"Row 121"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:27"}},"data":"Data for row 122","id":"Row 122"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:28"}},"data":"Data for row 123","id":"Row 123"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:29"}},"data":"Data for row 124","id":"Row 124"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:30"}},"data":"Data for row 125","id":"Row 125"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:31"}},"data":"Data for row 126","id":"Row 126"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:32"}},"data":"Data for row 127","id":"Row 127"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:33"}},"data":"Data for row 128","id":"Row 128"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:34"}},"data":"Data for row 129","id":"Row 129"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:35"}},"data":"Data for row 13","id":"Row 13"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:36"}},"data":"Data for row 130","id":"Row 130"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:37"}},"data":"Data for row 131","id":"Row 131"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:38"}},"data":"Data for row 132","id":"Row 132"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:39"}},"data":"Data for row 133","id":"Row 133"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:40"}},"data":"Data for row 134","id":"Row 134"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:41"}},"data":"Data for row 135","id":"Row 135"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:42"}},"data":"Data for row 136","id":"Row 136"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:43"}},"data":"Data for row 137","id":"Row 137"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:44"}},"data":"Data for row 138","id":"Row 138"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:45"}},"data":"Data for row 139","id":"Row 139"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:46"}},"data":"Data for row 14","id":"Row 14"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:47"}},"data":"Data for row 140","id":"Row 140"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:48"}},"data":"Data for row 141","id":"Row 141"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:49"}},"data":"Data for row 142","id":"Row 142"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:50"}},"data":"Data for row 143","id":"Row 143"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:51"}},"data":"Data for row 144","id":"Row 144"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:52"}},"data":"Data for row 145","id":"Row 145"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:53"}},"data":"Data for row 146","id":"Row 146"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:54"}},"data":"Data for row 147","id":"Row 147"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:55"}},"data":"Data for row 148","id":"Row 148"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:56"}},"data":"Data for row 149","id":"Row 149"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:57"}},"data":"Data for row 15","id":"Row 15"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:58"}},"data":"Data for row 150","id":"Row 150"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:59"}},"data":"Data for row 151","id":"Row 151"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:60"}},"data":"Data for row 152","id":"Row 152"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:61"}},"data":"Data for row 153","id":"Row 153"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:62"}},"data":"Data for row 154","id":"Row 154"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:63"}},"data":"Data for row 155","id":"Row 155"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:64"}},"data":"Data for row 156","id":"Row 156"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:65"}},"data":"Data for row 157","id":"Row 157"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:66"}},"data":"Data for row 158","id":"Row 158"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:67"}},"data":"Data for row 159","id":"Row 159"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:68"}},"data":"Data for row 16","id":"Row 16"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:69"}},"data":"Data for row 160","id":"Row 160"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:70"}},"data":"Data for row 161","id":"Row 161"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:71"}},"data":"Data for row 162","id":"Row 162"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:72"}},"data":"Data for row 163","id":"Row 163"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:73"}},"data":"Data for row 164","id":"Row 164"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:74"}},"data":"Data for row 165","id":"Row 165"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:75"}},"data":"Data for row 166","id":"Row 166"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:76"}},"data":"Data for row 167","id":"Row 167"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:77"}},"data":"Data for row 168","id":"Row 168"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:78"}},"data":"Data for row 169","id":"Row 169"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:79"}},"data":"Data for row 17","id":"Row 17"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:80"}},"data":"Data for row 170","id":"Row 170"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:81"}},"data":"Data for row 171","id":"Row 171"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:82"}},"data":"Data for row 172","id":"Row 172"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:83"}},"data":"Data for row 173","id":"Row 173"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:84"}},"data":"Data for row 174","id":"Row 174"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:85"}},"data":"Data for row 175","id":"Row 175"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:86"}},"data":"Data for row 176","id":"Row 176"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:87"}},"data":"Data for row 177","id":"Row 177"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:88"}},"data":"Data for row 178","id":"Row 178"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:89"}},"data":"Data for row 179","id":"Row 179"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:90"}},"data":"Data for row 18","id":"Row 18"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:91"}},"data":"Data for row 180","id":"Row 180"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:92"}},"data":"Data for row 181","id":"Row 181"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:93"}},"data":"Data for row 182","id":"Row 182"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:94"}},"data":"Data for row 183","id":"Row 183"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:95"}},"data":"Data for row 184","id":"Row 184"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:96"}},"data":"Data for row 185","id":"Row 185"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:97"}},"data":"Data for row 186","id":"Row 186"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:98"}},"data":"Data for row 187","id":"Row 187"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:99"}},"data":"Data for row 188","id":"Row 188"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:100"}},"data":"Data for row 189","id":"Row 189"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:101"}},"data":"Data for row 19","id":"Row 19"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:102"}},"data":"Data for row 190","id":"Row 190"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:103"}},"data":"Data for row 191","id":"Row 191"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:104"}},"data":"Data for row 192","id":"Row 192"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:105"}},"data":"Data for row 193","id":"Row 193"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:106"}},"data":"Data for row 194","id":"Row 194"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:107"}},"data":"Data for row 195","id":"Row 195"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:108"}},"data":"Data for row 196","id":"Row 196"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:109"}},"data":"Data for row 197","id":"Row 197"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:110"}},"data":"Data for row 198","id":"Row 198"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:111"}},"data":"Data for row 199","id":"Row 199"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:112"}},"data":"Data for row 2","id":"Row 2"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:113"}},"data":"Data for row 20","id":"Row 20"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:114"}},"data":"Data for row 200","id":"Row 200"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:115"}},"data":"Data for row 201","id":"Row 201"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:116"}},"data":"Data for row 202","id":"Row 202"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:117"}},"data":"Data for row 203","id":"Row 203"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:118"}},"data":"Data for row 204","id":"Row 204"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:119"}},"data":"Data for row 205","id":"Row 205"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:120"}},"data":"Data for row 206","id":"Row 206"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:121"}},"data":"Data for row 207","id":"Row 207"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:122"}},"data":"Data for row 208","id":"Row 208"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:123"}},"data":"Data for row 209","id":"Row 209"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:124"}},"data":"Data for row 21","id":"Row 21"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:125"}},"data":"Data for row 210","id":"Row 210"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:126"}},"data":"Data for row 211","id":"Row 211"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:127"}},"data":"Data for row 212","id":"Row 212"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:128"}},"data":"Data for row 213","id":"Row 213"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:129"}},"data":"Data for row 214","id":"Row 214"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:130"}},"data":"Data for row 215","id":"Row 215"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:131"}},"data":"Data for row 216","id":"Row 216"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:132"}},"data":"Data for row 217","id":"Row 217"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:133"}},"data":"Data for row 218","id":"Row 218"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:134"}},"data":"Data for row 219","id":"Row 219"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:135"}},"data":"Data for row 22","id":"Row 22"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:136"}},"data":"Data for row 220","id":"Row 220"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:137"}},"data":"Data for row 221","id":"Row 221"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:138"}},"data":"Data for row 222","id":"Row 222"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:139"}},"data":"Data for row 223","id":"Row 223"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:140"}},"data":"Data for row 224","id":"Row 224"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:141"}},"data":"Data for row 225","id":"Row 225"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:142"}},"data":"Data for row 226","id":"Row 226"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:143"}},"data":"Data for row 227","id":"Row 227"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:144"}},"data":"Data for row 228","id":"Row 228"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:145"}},"data":"Data for row 229","id":"Row 229"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:146"}},"data":"Data for row 23","id":"Row 23"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:147"}},"data":"Data for row 230","id":"Row 230"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:148"}},"data":"Data for row 231","id":"Row 231"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:149"}},"data":"Data for row 232","id":"Row 232"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:150"}},"data":"Data for row 233","id":"Row 233"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:151"}},"data":"Data for row 234","id":"Row 234"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:152"}},"data":"Data for row 235","id":"Row 235"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:153"}},"data":"Data for row 236","id":"Row 236"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:154"}},"data":"Data for row 237","id":"Row 237"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:155"}},"data":"Data for row 238","id":"Row 238"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:156"}},"data":"Data for row 239","id":"Row 239"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:157"}},"data":"Data for row 24","id":"Row 24"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:158"}},"data":"Data for row 240","id":"Row 240"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:159"}},"data":"Data for row 241","id":"Row 241"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:160"}},"data":"Data for row 242","id":"Row 242"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:161"}},"data":"Data for row 243","id":"Row 243"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:162"}},"data":"Data for row 244","id":"Row 244"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:163"}},"data":"Data for row 245","id":"Row 245"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:164"}},"data":"Data for row 246","id":"Row 246"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:165"}},"data":"Data for row 247","id":"Row 247"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:166"}},"data":"Data for row 248","id":"Row 248"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:167"}},"data":"Data for row 249","id":"Row 249"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:168"}},"data":"Data for row 25","id":"Row 25"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:169"}},"data":"Data for row 250","id":"Row 250"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:170"}},"data":"Data for row 251","id":"Row 251"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:171"}},"data":"Data for row 252","id":"Row 252"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:172"}},"data":"Data for row 253","id":"Row 253"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:173"}},"data":"Data for row 254","id":"Row 254"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:174"}},"data":"Data for row 255","id":"Row 255"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:175"}},"data":"Data for row 256","id":"Row 256"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:176"}},"data":"Data for row 257","id":"Row 257"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:177"}},"data":"Data for row 258","id":"Row 258"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:178"}},"data":"Data for row 259","id":"Row 259"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:179"}},"data":"Data for row 26","id":"Row 26"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:180"}},"data":"Data for row 260","id":"Row 260"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:181"}},"data":"Data for row 261","id":"Row 261"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:182"}},"data":"Data for row 262","id":"Row 262"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:183"}},"data":"Data for row 263","id":"Row 263"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:184"}},"data":"Data for row 264","id":"Row 264"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:185"}},"data":"Data for row 265","id":"Row 265"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:186"}},"data":"Data for row 266","id":"Row 266"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:187"}},"data":"Data for row 267","id":"Row 267"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:188"}},"data":"Data for row 268","id":"Row 268"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:189"}},"data":"Data for row 269","id":"Row 269"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:190"}},"data":"Data for row 27","id":"Row 27"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:191"}},"data":"Data for row 270","id":"Row 270"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:192"}},"data":"Data for row 271","id":"Row 271"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:193"}},"data":"Data for row 272","id":"Row 272"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:194"}},"data":"Data for row 273","id":"Row 273"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:195"}},"data":"Data for row 274","id":"Row 274"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:196"}},"data":"Data for row 275","id":"Row 275"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:197"}},"data":"Data for row 276","id":"Row 276"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:198"}},"data":"Data for row 277","id":"Row 277"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:199"}},"data":"Data for row 278","id":"Row 278"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:200"}},"data":"Data for row 279","id":"Row 279"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:201"}},"data":"Data for row 28","id":"Row 28"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:202"}},"data":"Data for row 280","id":"Row 280"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:203"}},"data":"Data for row 281","id":"Row 281"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:204"}},"data":"Data for row 282","id":"Row 282"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:205"}},"data":"Data for row 283","id":"Row 283"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:206"}},"data":"Data for row 284","id":"Row 284"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:207"}},"data":"Data for row 285","id":"Row 285"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:208"}},"data":"Data for row 286","id":"Row 286"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:209"}},"data":"Data for row 287","id":"Row 287"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:210"}},"data":"Data for row 288","id":"Row 288"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:211"}},"data":"Data for row 289","id":"Row 289"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:212"}},"data":"Data for row 29","id":"Row 29"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:213"}},"data":"Data for row 290","id":"Row 290"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:214"}},"data":"Data for row 291","id":"Row 291"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:215"}},"data":"Data for row 292","id":"Row 292"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:216"}},"data":"Data for row 293","id":"Row 293"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:217"}},"data":"Data for row 294","id":"Row 294"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:218"}},"data":"Data for row 295","id":"Row 295"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:219"}},"data":"Data for row 296","id":"Row 296"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:220"}},"data":"Data for row 297","id":"Row 297"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:221"}},"data":"Data for row 298","id":"Row 298"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:222"}},"data":"Data for row 299","id":"Row 299"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:223"}},"data":"Data for row 3","id":"Row 3"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:224"}},"data":"Data for row 30","id":"Row 30"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:225"}},"data":"Data for row 300","id":"Row 300"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:226"}},"data":"Data for row 301","id":"Row 301"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:227"}},"data":"Data for row 302","id":"Row 302"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:228"}},"data":"Data for row 303","id":"Row 303"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:229"}},"data":"Data for row 304","id":"Row 304"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:230"}},"data":"Data for row 305","id":"Row 305"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:231"}},"data":"Data for row 306","id":"Row 306"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:232"}},"data":"Data for row 307","id":"Row 307"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:233"}},"data":"Data for row 308","id":"Row 308"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:234"}},"data":"Data for row 309","id":"Row 309"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:235"}},"data":"Data for row 31","id":"Row 31"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:236"}},"data":"Data for row 310","id":"Row 310"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:237"}},"data":"Data for row 311","id":"Row 311"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:238"}},"data":"Data for row 312","id":"Row 312"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:239"}},"data":"Data for row 313","id":"Row 313"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:240"}},"data":"Data for row 314","id":"Row 314"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:241"}},"data":"Data for row 315","id":"Row 315"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:242"}},"data":"Data for row 316","id":"Row 316"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:243"}},"data":"Data for row 317","id":"Row 317"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:244"}},"data":"Data for row 318","id":"Row 318"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:245"}},"data":"Data for row 319","id":"Row 319"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:246"}},"data":"Data for row 32","id":"Row 32"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:247"}},"data":"Data for row 320","id":"Row 320"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:248"}},"data":"Data for row 321","id":"Row 321"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:249"}},"data":"Data for row 322","id":"Row 322"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:250"}},"data":"Data for row 323","id":"Row 323"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:251"}},"data":"Data for row 324","id":"Row 324"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:252"}},"data":"Data for row 325","id":"Row 325"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:253"}},"data":"Data for row 326","id":"Row 326"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:254"}},"data":"Data for row 327","id":"Row 327"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:255"}},"data":"Data for row 328","id":"Row 328"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:256"}},"data":"Data for row 329","id":"Row 329"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:257"}},"data":"Data for row 33","id":"Row 33"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:258"}},"data":"Data for row 330","id":"Row 330"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:259"}},"data":"Data for row 331","id":"Row 331"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:260"}},"data":"Data for row 332","id":"Row 332"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:261"}},"data":"Data for row 333","id":"Row 333"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:262"}},"data":"Data for row 334","id":"Row 334"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:263"}},"data":"Data for row 335","id":"Row 335"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:264"}},"data":"Data for row 336","id":"Row 336"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:265"}},"data":"Data for row 337","id":"Row 337"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:266"}},"data":"Data for row 338","id":"Row 338"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:267"}},"data":"Data for row 339","id":"Row 339"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:268"}},"data":"Data for row 34","id":"Row 34"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:269"}},"data":"Data for row 340","id":"Row 340"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:270"}},"data":"Data for row 341","id":"Row 341"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:271"}},"data":"Data for row 342","id":"Row 342"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:272"}},"data":"Data for row 343","id":"Row 343"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:273"}},"data":"Data for row 344","id":"Row 344"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:274"}},"data":"Data for row 345","id":"Row 345"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:275"}},"data":"Data for row 346","id":"Row 346"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:276"}},"data":"Data for row 347","id":"Row 347"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:277"}},"data":"Data for row 348","id":"Row 348"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:278"}},"data":"Data for row 349","id":"Row 349"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:279"}},"data":"Data for row 35","id":"Row 35"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:280"}},"data":"Data for row 350","id":"Row 350"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:281"}},"data":"Data for row 351","id":"Row 351"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:282"}},"data":"Data for row 352","id":"Row 352"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:283"}},"data":"Data for row 353","id":"Row 353"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:284"}},"data":"Data for row 354","id":"Row 354"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:285"}},"data":"Data for row 355","id":"Row 355"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:286"}},"data":"Data for row 356","id":"Row 356"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:287"}},"data":"Data for row 357","id":"Row 357"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:288"}},"data":"Data for row 358","id":"Row 358"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:289"}},"data":"Data for row 359","id":"Row 359"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:290"}},"data":"Data for row 36","id":"Row 36"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:291"}},"data":"Data for row 360","id":"Row 360"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:292"}},"data":"Data for row 361","id":"Row 361"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:293"}},"data":"Data for row 362","id":"Row 362"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:294"}},"data":"Data for row 363","id":"Row 363"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:295"}},"data":"Data for row 364","id":"Row 364"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:296"}},"data":"Data for row 365","id":"Row 365"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:297"}},"data":"Data for row 366","id":"Row 366"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:298"}},"data":"Data for row 367","id":"Row 367"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:299"}},"data":"Data for row 368","id":"Row 368"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:300"}},"data":"Data for row 369","id":"Row 369"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:301"}},"data":"Data for row 37","id":"Row 37"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:302"}},"data":"Data for row 370","id":"Row 370"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:303"}},"data":"Data for row 371","id":"Row 371"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:304"}},"data":"Data for row 372","id":"Row 372"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:305"}},"data":"Data for row 373","id":"Row 373"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:306"}},"data":"Data for row 374","id":"Row 374"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:307"}},"data":"Data for row 375","id":"Row 375"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:308"}},"data":"Data for row 376","id":"Row 376"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:309"}},"data":"Data for row 377","id":"Row 377"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:310"}},"data":"Data for row 378","id":"Row 378"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:311"}},"data":"Data for row 379","id":"Row 379"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:312"}},"data":"Data for row 38","id":"Row 38"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:313"}},"data":"Data for row 380","id":"Row 380"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:314"}},"data":"Data for row 381","id":"Row 381"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:315"}},"data":"Data for row 382","id":"Row 382"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:316"}},"data":"Data for row 383","id":"Row 383"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:317"}},"data":"Data for row 384","id":"Row 384"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:318"}},"data":"Data for row 385","id":"Row 385"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:319"}},"data":"Data for row 386","id":"Row 386"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:320"}},"data":"Data for row 387","id":"Row 387"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:321"}},"data":"Data for row 388","id":"Row 388"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:322"}},"data":"Data for row 389","id":"Row 389"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:323"}},"data":"Data for row 39","id":"Row 39"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:324"}},"data":"Data for row 390","id":"Row 390"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:325"}},"data":"Data for row 391","id":"Row 391"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:326"}},"data":"Data for row 392","id":"Row 392"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:327"}},"data":"Data for row 393","id":"Row 393"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:328"}},"data":"Data for row 394","id":"Row 394"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:329"}},"data":"Data for row 395","id":"Row 395"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:330"}},"data":"Data for row 396","id":"Row 396"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:331"}},"data":"Data for row 397","id":"Row 397"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:332"}},"data":"Data for row 398","id":"Row 398"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:333"}},"data":"Data for row 399","id":"Row 399"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:334"}},"data":"Data for row 4","id":"Row 4"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:335"}},"data":"Data for row 40","id":"Row 40"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:336"}},"data":"Data for row 400","id":"Row 400"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:337"}},"data":"Data for row 401","id":"Row 401"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:338"}},"data":"Data for row 402","id":"Row 402"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:339"}},"data":"Data for row 403","id":"Row 403"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:340"}},"data":"Data for row 404","id":"Row 404"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:341"}},"data":"Data for row 405","id":"Row 405"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:342"}},"data":"Data for row 406","id":"Row 406"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:343"}},"data":"Data for row 407","id":"Row 407"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:344"}},"data":"Data for row 408","id":"Row 408"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:345"}},"data":"Data for row 409","id":"Row 409"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:346"}},"data":"Data for row 41","id":"Row 41"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:347"}},"data":"Data for row 410","id":"Row 410"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:348"}},"data":"Data for row 411","id":"Row 411"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:349"}},"data":"Data for row 412","id":"Row 412"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:350"}},"data":"Data for row 413","id":"Row 413"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:351"}},"data":"Data for row 414","id":"Row 414"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:352"}},"data":"Data for row 415","id":"Row 415"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:353"}},"data":"Data for row 416","id":"Row 416"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:354"}},"data":"Data for row 417","id":"Row 417"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:355"}},"data":"Data for row 418","id":"Row 418"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:356"}},"data":"Data for row 419","id":"Row 419"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:357"}},"data":"Data for row 42","id":"Row 42"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:358"}},"data":"Data for row 420","id":"Row 420"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:359"}},"data":"Data for row 421","id":"Row 421"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:360"}},"data":"Data for row 422","id":"Row 422"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:361"}},"data":"Data for row 423","id":"Row 423"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:362"}},"data":"Data for row 424","id":"Row 424"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:363"}},"data":"Data for row 425","id":"Row 425"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:364"}},"data":"Data for row 426","id":"Row 426"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:365"}},"data":"Data for row 427","id":"Row 427"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:366"}},"data":"Data for row 428","id":"Row 428"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:367"}},"data":"Data for row 429","id":"Row 429"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:368"}},"data":"Data for row 43","id":"Row 43"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:369"}},"data":"Data for row 430","id":"Row 430"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:370"}},"data":"Data for row 431","id":"Row 431"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:371"}},"data":"Data for row 432","id":"Row 432"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:372"}},"data":"Data for row 433","id":"Row 433"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:373"}},"data":"Data for row 434","id":"Row 434"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:374"}},"data":"Data for row 435","id":"Row 435"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:375"}},"data":"Data for row 436","id":"Row 436"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:376"}},"data":"Data for row 437","id":"Row 437"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:377"}},"data":"Data for row 438","id":"Row 438"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:378"}},"data":"Data for row 439","id":"Row 439"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:379"}},"data":"Data for row 44","id":"Row 44"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:380"}},"data":"Data for row 440","id":"Row 440"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:381"}},"data":"Data for row 441","id":"Row 441"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:382"}},"data":"Data for row 442","id":"Row 442"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:383"}},"data":"Data for row 443","id":"Row 443"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:384"}},"data":"Data for row 444","id":"Row 444"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:385"}},"data":"Data for row 445","id":"Row 445"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:386"}},"data":"Data for row 446","id":"Row 446"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:387"}},"data":"Data for row 447","id":"Row 447"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:388"}},"data":"Data for row 448","id":"Row 448"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:389"}},"data":"Data for row 449","id":"Row 449"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:390"}},"data":"Data for row 45","id":"Row 45"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:391"}},"data":"Data for row 450","id":"Row 450"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:392"}},"data":"Data for row 451","id":"Row 451"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:393"}},"data":"Data for row 452","id":"Row 452"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:394"}},"data":"Data for row 453","id":"Row 453"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:395"}},"data":"Data for row 454","id":"Row 454"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:396"}},"data":"Data for row 455","id":"Row 455"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:397"}},"data":"Data for row 456","id":"Row 456"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:398"}},"data":"Data for row 457","id":"Row 457"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:399"}},"data":"Data for row 458","id":"Row 458"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:400"}},"data":"Data for row 459","id":"Row 459"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:401"}},"data":"Data for row 46","id":"Row 46"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:402"}},"data":"Data for row 460","id":"Row 460"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:403"}},"data":"Data for row 461","id":"Row 461"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:404"}},"data":"Data for row 462","id":"Row 462"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:405"}},"data":"Data for row 463","id":"Row 463"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:406"}},"data":"Data for row 464","id":"Row 464"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:407"}},"data":"Data for row 465","id":"Row 465"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:408"}},"data":"Data for row 466","id":"Row 466"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:409"}},"data":"Data for row 467","id":"Row 467"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:410"}},"data":"Data for row 468","id":"Row 468"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:411"}},"data":"Data for row 469","id":"Row 469"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:412"}},"data":"Data for row 47","id":"Row 47"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:413"}},"data":"Data for row 470","id":"Row 470"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:414"}},"data":"Data for row 471","id":"Row 471"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:415"}},"data":"Data for row 472","id":"Row 472"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:416"}},"data":"Data for row 473","id":"Row 473"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:417"}},"data":"Data for row 474","id":"Row 474"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:418"}},"data":"Data for row 475","id":"Row 475"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:419"}},"data":"Data for row 476","id":"Row 476"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:420"}},"data":"Data for row 477","id":"Row 477"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:421"}},"data":"Data for row 478","id":"Row 478"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:422"}},"data":"Data for row 479","id":"Row 479"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:423"}},"data":"Data for row 48","id":"Row 48"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:424"}},"data":"Data for row 480","id":"Row 480"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:425"}},"data":"Data for row 481","id":"Row 481"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:426"}},"data":"Data for row 482","id":"Row 482"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:427"}},"data":"Data for row 483","id":"Row 483"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:428"}},"data":"Data for row 484","id":"Row 484"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:429"}},"data":"Data for row 485","id":"Row 485"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:430"}},"data":"Data for row 486","id":"Row 486"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:431"}},"data":"Data for row 487","id":"Row 487"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:432"}},"data":"Data for row 488","id":"Row 488"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:433"}},"data":"Data for row 489","id":"Row 489"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:434"}},"data":"Data for row 49","id":"Row 49"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:435"}},"data":"Data for row 490","id":"Row 490"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:436"}},"data":"Data for row 491","id":"Row 491"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:437"}},"data":"Data for row 492","id":"Row 492"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:438"}},"data":"Data for row 493","id":"Row 493"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:439"}},"data":"Data for row 494","id":"Row 494"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:440"}},"data":"Data for row 495","id":"Row 495"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:441"}},"data":"Data for row 496","id":"Row 496"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:442"}},"data":"Data for row 497","id":"Row 497"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:443"}},"data":"Data for row 498","id":"Row 498"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:444"}},"data":"Data for row 499","id":"Row 499"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:445"}},"data":"Data for row 5","id":"Row 5"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:446"}},"data":"Data for row 50","id":"Row 50"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:447"}},"data":"Data for row 500","id":"Row 500"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:448"}},"data":"Data for row 501","id":"Row 501"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:449"}},"data":"Data for row 502","id":"Row 502"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:450"}},"data":"Data for row 503","id":"Row 503"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:451"}},"data":"Data for row 504","id":"Row 504"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:452"}},"data":"Data for row 505","id":"Row 505"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:453"}},"data":"Data for row 506","id":"Row 506"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:454"}},"data":"Data for row 507","id":"Row 507"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:455"}},"data":"Data for row 508","id":"Row 508"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:456"}},"data":"Data for row 509","id":"Row 509"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:457"}},"data":"Data for row 51","id":"Row 51"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:458"}},"data":"Data for row 510","id":"Row 510"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:459"}},"data":"Data for row 511","id":"Row 511"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:460"}},"data":"Data for row 512","id":"Row 512"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:461"}},"data":"Data for row 513","id":"Row 513"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:462"}},"data":"Data for row 514","id":"Row 514"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:463"}},"data":"Data for row 515","id":"Row 515"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:464"}},"data":"Data for row 516","id":"Row 516"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:465"}},"data":"Data for row 517","id":"Row 517"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:466"}},"data":"Data for row 518","id":"Row 518"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:467"}},"data":"Data for row 519","id":"Row 519"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:468"}},"data":"Data for row 52","id":"Row 52"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:469"}},"data":"Data for row 520","id":"Row 520"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:470"}},"data":"Data for row 521","id":"Row 521"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:471"}},"data":"Data for row 522","id":"Row 522"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:472"}},"data":"Data for row 523","id":"Row 523"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:473"}},"data":"Data for row 524","id":"Row 524"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:474"}},"data":"Data for row 525","id":"Row 525"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:475"}},"data":"Data for row 526","id":"Row 526"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:476"}},"data":"Data for row 527","id":"Row 527"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:477"}},"data":"Data for row 528","id":"Row 528"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:478"}},"data":"Data for row 529","id":"Row 529"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:479"}},"data":"Data for row 53","id":"Row 53"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:480"}},"data":"Data for row 530","id":"Row 530"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:481"}},"data":"Data for row 531","id":"Row 531"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:482"}},"data":"Data for row 532","id":"Row 532"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:483"}},"data":"Data for row 533","id":"Row 533"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:484"}},"data":"Data for row 534","id":"Row 534"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:485"}},"data":"Data for row 535","id":"Row 535"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:486"}},"data":"Data for row 536","id":"Row 536"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:487"}},"data":"Data for row 537","id":"Row 537"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:488"}},"data":"Data for row 538","id":"Row 538"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:489"}},"data":"Data for row 539","id":"Row 539"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:490"}},"data":"Data for row 54","id":"Row 54"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:491"}},"data":"Data for row 540","id":"Row 540"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:492"}},"data":"Data for row 541","id":"Row 541"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:493"}},"data":"Data for row 542","id":"Row 542"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:494"}},"data":"Data for row 543","id":"Row 543"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:495"}},"data":"Data for row 544","id":"Row 544"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:496"}},"data":"Data for row 545","id":"Row 545"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:497"}},"data":"Data for row 546","id":"Row 546"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:498"}},"data":"Data for row 547","id":"Row 547"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:499"}},"data":"Data for row 548","id":"Row 548"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:500"}},"data":"Data for row 549","id":"Row 549"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:501"}},"data":"Data for row 55","id":"Row 55"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:502"}},"data":"Data for row 550","id":"Row 550"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:503"}},"data":"Data for row 551","id":"Row 551"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:504"}},"data":"Data for row 552","id":"Row 552"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:505"}},"data":"Data for row 553","id":"Row 553"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:506"}},"data":"Data for row 554","id":"Row 554"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:507"}},"data":"Data for row 555","id":"Row 555"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:508"}},"data":"Data for row 556","id":"Row 556"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:509"}},"data":"Data for row 557","id":"Row 557"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:510"}},"data":"Data for row 558","id":"Row 558"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:511"}},"data":"Data for row 559","id":"Row 559"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:512"}},"data":"Data for row 56","id":"Row 56"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:513"}},"data":"Data for row 560","id":"Row 560"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:514"}},"data":"Data for row 561","id":"Row 561"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:515"}},"data":"Data for row 562","id":"Row 562"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:516"}},"data":"Data for row 563","id":"Row 563"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:517"}},"data":"Data for row 564","id":"Row 564"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:518"}},"data":"Data for row 565","id":"Row 565"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:519"}},"data":"Data for row 566","id":"Row 566"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:520"}},"data":"Data for row 567","id":"Row 567"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:521"}},"data":"Data for row 568","id":"Row 568"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:522"}},"data":"Data for row 569","id":"Row 569"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:523"}},"data":"Data for row 57","id":"Row 57"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:524"}},"data":"Data for row 570","id":"Row 570"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:525"}},"data":"Data for row 571","id":"Row 571"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:526"}},"data":"Data for row 572","id":"Row 572"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:527"}},"data":"Data for row 573","id":"Row 573"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:528"}},"data":"Data for row 574","id":"Row 574"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:529"}},"data":"Data for row 575","id":"Row 575"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:530"}},"data":"Data for row 576","id":"Row 576"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:531"}},"data":"Data for row 577","id":"Row 577"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:532"}},"data":"Data for row 578","id":"Row 578"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:533"}},"data":"Data for row 579","id":"Row 579"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:534"}},"data":"Data for row 58","id":"Row 58"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:535"}},"data":"Data for row 580","id":"Row 580"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:536"}},"data":"Data for row 581","id":"Row 581"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:537"}},"data":"Data for row 582","id":"Row 582"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:538"}},"data":"Data for row 583","id":"Row 583"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:539"}},"data":"Data for row 584","id":"Row 584"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:540"}},"data":"Data for row 585","id":"Row 585"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:541"}},"data":"Data for row 586","id":"Row 586"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:542"}},"data":"Data for row 587","id":"Row 587"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:543"}},"data":"Data for row 588","id":"Row 588"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:544"}},"data":"Data for row 589","id":"Row 589"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:545"}},"data":"Data for row 59","id":"Row 59"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:546"}},"data":"Data for row 590","id":"Row 590"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:547"}},"data":"Data for row 591","id":"Row 591"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:548"}},"data":"Data for row 592","id":"Row 592"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:549"}},"data":"Data for row 593","id":"Row 593"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:550"}},"data":"Data for row 594","id":"Row 594"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:551"}},"data":"Data for row 595","id":"Row 595"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:552"}},"data":"Data for row 596","id":"Row 596"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:553"}},"data":"Data for row 597","id":"Row 597"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:554"}},"data":"Data for row 598","id":"Row 598"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:555"}},"data":"Data for row 599","id":"Row 599"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:556"}},"data":"Data for row 6","id":"Row 6"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:557"}},"data":"Data for row 60","id":"Row 60"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:558"}},"data":"Data for row 600","id":"Row 600"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:559"}},"data":"Data for row 601","id":"Row 601"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:560"}},"data":"Data for row 602","id":"Row 602"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:561"}},"data":"Data for row 603","id":"Row 603"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:562"}},"data":"Data for row 604","id":"Row 604"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:563"}},"data":"Data for row 605","id":"Row 605"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:564"}},"data":"Data for row 606","id":"Row 606"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:565"}},"data":"Data for row 607","id":"Row 607"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:566"}},"data":"Data for row 608","id":"Row 608"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:567"}},"data":"Data for row 609","id":"Row 609"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:568"}},"data":"Data for row 61","id":"Row 61"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:569"}},"data":"Data for row 610","id":"Row 610"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:570"}},"data":"Data for row 611","id":"Row 611"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:571"}},"data":"Data for row 612","id":"Row 612"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:572"}},"data":"Data for row 613","id":"Row 613"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:573"}},"data":"Data for row 614","id":"Row 614"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:574"}},"data":"Data for row 615","id":"Row 615"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:575"}},"data":"Data for row 616","id":"Row 616"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:576"}},"data":"Data for row 617","id":"Row 617"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:577"}},"data":"Data for row 618","id":"Row 618"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:578"}},"data":"Data for row 619","id":"Row 619"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:579"}},"data":"Data for row 62","id":"Row 62"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:580"}},"data":"Data for row 620","id":"Row 620"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:581"}},"data":"Data for row 621","id":"Row 621"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:582"}},"data":"Data for row 622","id":"Row 622"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:583"}},"data":"Data for row 623","id":"Row 623"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:584"}},"data":"Data for row 624","id":"Row 624"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:585"}},"data":"Data for row 625","id":"Row 625"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:586"}},"data":"Data for row 626","id":"Row 626"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:587"}},"data":"Data for row 627","id":"Row 627"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:588"}},"data":"Data for row 628","id":"Row 628"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:589"}},"data":"Data for row 629","id":"Row 629"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:590"}},"data":"Data for row 63","id":"Row 63"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:591"}},"data":"Data for row 630","id":"Row 630"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:592"}},"data":"Data for row 631","id":"Row 631"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:593"}},"data":"Data for row 632","id":"Row 632"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:594"}},"data":"Data for row 633","id":"Row 633"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:595"}},"data":"Data for row 634","id":"Row 634"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:596"}},"data":"Data for row 635","id":"Row 635"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:597"}},"data":"Data for row 636","id":"Row 636"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:598"}},"data":"Data for row 637","id":"Row 637"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:599"}},"data":"Data for row 638","id":"Row 638"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:600"}},"data":"Data for row 639","id":"Row 639"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:601"}},"data":"Data for row 64","id":"Row 64"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:602"}},"data":"Data for row 640","id":"Row 640"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:603"}},"data":"Data for row 641","id":"Row 641"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:604"}},"data":"Data for row 642","id":"Row 642"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:605"}},"data":"Data for row 643","id":"Row 643"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:606"}},"data":"Data for row 644","id":"Row 644"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:607"}},"data":"Data for row 645","id":"Row 645"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:608"}},"data":"Data for row 646","id":"Row 646"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:609"}},"data":"Data for row 647","id":"Row 647"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:610"}},"data":"Data for row 648","id":"Row 648"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:611"}},"data":"Data for row 649","id":"Row 649"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:612"}},"data":"Data for row 65","id":"Row 65"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:613"}},"data":"Data for row 650","id":"Row 650"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:614"}},"data":"Data for row 651","id":"Row 651"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:615"}},"data":"Data for row 652","id":"Row 652"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:616"}},"data":"Data for row 653","id":"Row 653"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:617"}},"data":"Data for row 654","id":"Row 654"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:618"}},"data":"Data for row 655","id":"Row 655"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:619"}},"data":"Data for row 656","id":"Row 656"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:620"}},"data":"Data for row 657","id":"Row 657"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:621"}},"data":"Data for row 658","id":"Row 658"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:622"}},"data":"Data for row 659","id":"Row 659"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:623"}},"data":"Data for row 66","id":"Row 66"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:624"}},"data":"Data for row 660","id":"Row 660"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:625"}},"data":"Data for row 661","id":"Row 661"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:626"}},"data":"Data for row 662","id":"Row 662"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:627"}},"data":"Data for row 663","id":"Row 663"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:628"}},"data":"Data for row 664","id":"Row 664"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:629"}},"data":"Data for row 665","id":"Row 665"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:630"}},"data":"Data for row 666","id":"Row 666"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:631"}},"data":"Data for row 667","id":"Row 667"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:632"}},"data":"Data for row 668","id":"Row 668"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:633"}},"data":"Data for row 669","id":"Row 669"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:634"}},"data":"Data for row 67","id":"Row 67"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:635"}},"data":"Data for row 670","id":"Row 670"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:636"}},"data":"Data for row 671","id":"Row 671"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:637"}},"data":"Data for row 672","id":"Row 672"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:638"}},"data":"Data for row 673","id":"Row 673"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:639"}},"data":"Data for row 674","id":"Row 674"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:640"}},"data":"Data for row 675","id":"Row 675"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:641"}},"data":"Data for row 676","id":"Row 676"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:642"}},"data":"Data for row 677","id":"Row 677"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:643"}},"data":"Data for row 678","id":"Row 678"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:644"}},"data":"Data for row 679","id":"Row 679"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:645"}},"data":"Data for row 68","id":"Row 68"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:646"}},"data":"Data for row 680","id":"Row 680"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:647"}},"data":"Data for row 681","id":"Row 681"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:648"}},"data":"Data for row 682","id":"Row 682"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:649"}},"data":"Data for row 683","id":"Row 683"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:650"}},"data":"Data for row 684","id":"Row 684"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:651"}},"data":"Data for row 685","id":"Row 685"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:652"}},"data":"Data for row 686","id":"Row 686"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:653"}},"data":"Data for row 687","id":"Row 687"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:654"}},"data":"Data for row 688","id":"Row 688"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:655"}},"data":"Data for row 689","id":"Row 689"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:656"}},"data":"Data for row 69","id":"Row 69"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:657"}},"data":"Data for row 690","id":"Row 690"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:658"}},"data":"Data for row 691","id":"Row 691"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:659"}},"data":"Data for row 692","id":"Row 692"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:660"}},"data":"Data for row 693","id":"Row 693"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:661"}},"data":"Data for row 694","id":"Row 694"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:662"}},"data":"Data for row 695","id":"Row 695"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:663"}},"data":"Data for row 696","id":"Row 696"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:664"}},"data":"Data for row 697","id":"Row 697"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:665"}},"data":"Data for row 698","id":"Row 698"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:666"}},"data":"Data for row 699","id":"Row 699"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:667"}},"data":"Data for row 7","id":"Row 7"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:668"}},"data":"Data for row 70","id":"Row 70"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:669"}},"data":"Data for row 700","id":"Row 700"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:670"}},"data":"Data for row 701","id":"Row 701"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:671"}},"data":"Data for row 702","id":"Row 702"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:672"}},"data":"Data for row 703","id":"Row 703"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:673"}},"data":"Data for row 704","id":"Row 704"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:674"}},"data":"Data for row 705","id":"Row 705"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:675"}},"data":"Data for row 706","id":"Row 706"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:676"}},"data":"Data for row 707","id":"Row 707"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:677"}},"data":"Data for row 708","id":"Row 708"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:678"}},"data":"Data for row 709","id":"Row 709"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:679"}},"data":"Data for row 71","id":"Row 71"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:680"}},"data":"Data for row 710","id":"Row 710"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:681"}},"data":"Data for row 711","id":"Row 711"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:682"}},"data":"Data for row 712","id":"Row 712"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:683"}},"data":"Data for row 713","id":"Row 713"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:684"}},"data":"Data for row 714","id":"Row 714"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:685"}},"data":"Data for row 715","id":"Row 715"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:686"}},"data":"Data for row 716","id":"Row 716"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:687"}},"data":"Data for row 717","id":"Row 717"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:688"}},"data":"Data for row 718","id":"Row 718"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:689"}},"data":"Data for row 719","id":"Row 719"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:690"}},"data":"Data for row 72","id":"Row 72"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:691"}},"data":"Data for row 720","id":"Row 720"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:692"}},"data":"Data for row 721","id":"Row 721"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:693"}},"data":"Data for row 722","id":"Row 722"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:694"}},"data":"Data for row 723","id":"Row 723"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:695"}},"data":"Data for row 724","id":"Row 724"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:696"}},"data":"Data for row 725","id":"Row 725"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:697"}},"data":"Data for row 726","id":"Row 726"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:698"}},"data":"Data for row 727","id":"Row 727"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:699"}},"data":"Data for row 728","id":"Row 728"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:700"}},"data":"Data for row 729","id":"Row 729"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:701"}},"data":"Data for row 73","id":"Row 73"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:702"}},"data":"Data for row 730","id":"Row 730"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:703"}},"data":"Data for row 731","id":"Row 731"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:704"}},"data":"Data for row 732","id":"Row 732"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:705"}},"data":"Data for row 733","id":"Row 733"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:706"}},"data":"Data for row 734","id":"Row 734"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:707"}},"data":"Data for row 735","id":"Row 735"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:708"}},"data":"Data for row 736","id":"Row 736"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:709"}},"data":"Data for row 737","id":"Row 737"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:710"}},"data":"Data for row 738","id":"Row 738"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:711"}},"data":"Data for row 739","id":"Row 739"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:712"}},"data":"Data for row 74","id":"Row 74"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:713"}},"data":"Data for row 740","id":"Row 740"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:714"}},"data":"Data for row 741","id":"Row 741"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:715"}},"data":"Data for row 742","id":"Row 742"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:716"}},"data":"Data for row 743","id":"Row 743"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:717"}},"data":"Data for row 744","id":"Row 744"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:718"}},"data":"Data for row 745","id":"Row 745"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:719"}},"data":"Data for row 746","id":"Row 746"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:720"}},"data":"Data for row 747","id":"Row 747"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:721"}},"data":"Data for row 748","id":"Row 748"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:722"}},"data":"Data for row 749","id":"Row 749"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:723"}},"data":"Data for row 75","id":"Row 75"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:724"}},"data":"Data for row 750","id":"Row 750"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:725"}},"data":"Data for row 751","id":"Row 751"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:726"}},"data":"Data for row 752","id":"Row 752"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:727"}},"data":"Data for row 753","id":"Row 753"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:728"}},"data":"Data for row 754","id":"Row 754"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:729"}},"data":"Data for row 755","id":"Row 755"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:730"}},"data":"Data for row 756","id":"Row 756"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:731"}},"data":"Data for row 757","id":"Row 757"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:732"}},"data":"Data for row 758","id":"Row 758"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:733"}},"data":"Data for row 759","id":"Row 759"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:734"}},"data":"Data for row 76","id":"Row 76"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:735"}},"data":"Data for row 760","id":"Row 760"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:736"}},"data":"Data for row 761","id":"Row 761"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:737"}},"data":"Data for row 762","id":"Row 762"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:738"}},"data":"Data for row 763","id":"Row 763"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:739"}},"data":"Data for row 764","id":"Row 764"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:740"}},"data":"Data for row 765","id":"Row 765"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:741"}},"data":"Data for row 766","id":"Row 766"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:742"}},"data":"Data for row 767","id":"Row 767"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:743"}},"data":"Data for row 768","id":"Row 768"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:744"}},"data":"Data for row 769","id":"Row 769"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:745"}},"data":"Data for row 77","id":"Row 77"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:746"}},"data":"Data for row 770","id":"Row 770"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:747"}},"data":"Data for row 771","id":"Row 771"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:748"}},"data":"Data for row 772","id":"Row 772"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:749"}},"data":"Data for row 773","id":"Row 773"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:750"}},"data":"Data for row 774","id":"Row 774"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:751"}},"data":"Data for row 775","id":"Row 775"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:752"}},"data":"Data for row 776","id":"Row 776"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:753"}},"data":"Data for row 777","id":"Row 777"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:754"}},"data":"Data for row 778","id":"Row 778"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:755"}},"data":"Data for row 779","id":"Row 779"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:756"}},"data":"Data for row 78","id":"Row 78"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:757"}},"data":"Data for row 780","id":"Row 780"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:758"}},"data":"Data for row 781","id":"Row 781"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:759"}},"data":"Data for row 782","id":"Row 782"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:760"}},"data":"Data for row 783","id":"Row 783"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:761"}},"data":"Data for row 784","id":"Row 784"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:762"}},"data":"Data for row 785","id":"Row 785"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:763"}},"data":"Data for row 786","id":"Row 786"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:764"}},"data":"Data for row 787","id":"Row 787"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:765"}},"data":"Data for row 788","id":"Row 788"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:766"}},"data":"Data for row 789","id":"Row 789"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:767"}},"data":"Data for row 79","id":"Row 79"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:768"}},"data":"Data for row 790","id":"Row 790"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:769"}},"data":"Data for row 791","id":"Row 791"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:770"}},"data":"Data for row 792","id":"Row 792"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:771"}},"data":"Data for row 793","id":"Row 793"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:772"}},"data":"Data for row 794","id":"Row 794"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:773"}},"data":"Data for row 795","id":"Row 795"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:774"}},"data":"Data for row 796","id":"Row 796"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:775"}},"data":"Data for row 797","id":"Row 797"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:776"}},"data":"Data for row 798","id":"Row 798"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:777"}},"data":"Data for row 799","id":"Row 799"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:778"}},"data":"Data for row 8","id":"Row 8"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:779"}},"data":"Data for row 80","id":"Row 80"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:780"}},"data":"Data for row 800","id":"Row 800"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:781"}},"data":"Data for row 801","id":"Row 801"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:782"}},"data":"Data for row 802","id":"Row 802"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:783"}},"data":"Data for row 803","id":"Row 803"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:784"}},"data":"Data for row 804","id":"Row 804"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:785"}},"data":"Data for row 805","id":"Row 805"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:786"}},"data":"Data for row 806","id":"Row 806"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:787"}},"data":"Data for row 807","id":"Row 807"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:788"}},"data":"Data for row 808","id":"Row 808"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:789"}},"data":"Data for row 809","id":"Row 809"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:790"}},"data":"Data for row 81","id":"Row 81"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:791"}},"data":"Data for row 810","id":"Row 810"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:792"}},"data":"Data for row 811","id":"Row 811"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:793"}},"data":"Data for row 812","id":"Row 812"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:794"}},"data":"Data for row 813","id":"Row 813"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:795"}},"data":"Data for row 814","id":"Row 814"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:796"}},"data":"Data for row 815","id":"Row 815"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:797"}},"data":"Data for row 816","id":"Row 816"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:798"}},"data":"Data for row 817","id":"Row 817"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:799"}},"data":"Data for row 818","id":"Row 818"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:800"}},"data":"Data for row 819","id":"Row 819"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:801"}},"data":"Data for row 82","id":"Row 82"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:802"}},"data":"Data for row 820","id":"Row 820"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:803"}},"data":"Data for row 821","id":"Row 821"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:804"}},"data":"Data for row 822","id":"Row 822"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:805"}},"data":"Data for row 823","id":"Row 823"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:806"}},"data":"Data for row 824","id":"Row 824"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:807"}},"data":"Data for row 825","id":"Row 825"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:808"}},"data":"Data for row 826","id":"Row 826"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:809"}},"data":"Data for row 827","id":"Row 827"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:810"}},"data":"Data for row 828","id":"Row 828"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:811"}},"data":"Data for row 829","id":"Row 829"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:812"}},"data":"Data for row 83","id":"Row 83"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:813"}},"data":"Data for row 830","id":"Row 830"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:814"}},"data":"Data for row 831","id":"Row 831"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:815"}},"data":"Data for row 832","id":"Row 832"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:816"}},"data":"Data for row 833","id":"Row 833"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:817"}},"data":"Data for row 834","id":"Row 834"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:818"}},"data":"Data for row 835","id":"Row 835"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:819"}},"data":"Data for row 836","id":"Row 836"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:820"}},"data":"Data for row 837","id":"Row 837"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:821"}},"data":"Data for row 838","id":"Row 838"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:822"}},"data":"Data for row 839","id":"Row 839"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:823"}},"data":"Data for row 84","id":"Row 84"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:824"}},"data":"Data for row 840","id":"Row 840"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:825"}},"data":"Data for row 841","id":"Row 841"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:826"}},"data":"Data for row 842","id":"Row 842"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:827"}},"data":"Data for row 843","id":"Row 843"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:828"}},"data":"Data for row 844","id":"Row 844"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:829"}},"data":"Data for row 845","id":"Row 845"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:830"}},"data":"Data for row 846","id":"Row 846"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:831"}},"data":"Data for row 847","id":"Row 847"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:832"}},"data":"Data for row 848","id":"Row 848"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:833"}},"data":"Data for row 849","id":"Row 849"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:834"}},"data":"Data for row 85","id":"Row 85"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:835"}},"data":"Data for row 850","id":"Row 850"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:836"}},"data":"Data for row 851","id":"Row 851"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:837"}},"data":"Data for row 852","id":"Row 852"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:838"}},"data":"Data for row 853","id":"Row 853"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:839"}},"data":"Data for row 854","id":"Row 854"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:840"}},"data":"Data for row 855","id":"Row 855"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:841"}},"data":"Data for row 856","id":"Row 856"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:842"}},"data":"Data for row 857","id":"Row 857"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:843"}},"data":"Data for row 858","id":"Row 858"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:844"}},"data":"Data for row 859","id":"Row 859"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:845"}},"data":"Data for row 86","id":"Row 86"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:846"}},"data":"Data for row 860","id":"Row 860"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:847"}},"data":"Data for row 861","id":"Row 861"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:848"}},"data":"Data for row 862","id":"Row 862"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:849"}},"data":"Data for row 863","id":"Row 863"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:850"}},"data":"Data for row 864","id":"Row 864"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:851"}},"data":"Data for row 865","id":"Row 865"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:852"}},"data":"Data for row 866","id":"Row 866"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:853"}},"data":"Data for row 867","id":"Row 867"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:854"}},"data":"Data for row 868","id":"Row 868"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:855"}},"data":"Data for row 869","id":"Row 869"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:856"}},"data":"Data for row 87","id":"Row 87"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:857"}},"data":"Data for row 870","id":"Row 870"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:858"}},"data":"Data for row 871","id":"Row 871"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:859"}},"data":"Data for row 872","id":"Row 872"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:860"}},"data":"Data for row 873","id":"Row 873"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:861"}},"data":"Data for row 874","id":"Row 874"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:862"}},"data":"Data for row 875","id":"Row 875"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:863"}},"data":"Data for row 876","id":"Row 876"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:864"}},"data":"Data for row 877","id":"Row 877"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:865"}},"data":"Data for row 878","id":"Row 878"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:866"}},"data":"Data for row 879","id":"Row 879"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:867"}},"data":"Data for row 88","id":"Row 88"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:868"}},"data":"Data for row 880","id":"Row 880"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:869"}},"data":"Data for row 881","id":"Row 881"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:870"}},"data":"Data for row 882","id":"Row 882"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:871"}},"data":"Data for row 883","id":"Row 883"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:872"}},"data":"Data for row 884","id":"Row 884"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:873"}},"data":"Data for row 885","id":"Row 885"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:874"}},"data":"Data for row 886","id":"Row 886"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:875"}},"data":"Data for row 887","id":"Row 887"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:876"}},"data":"Data for row 888","id":"Row 888"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:877"}},"data":"Data for row 889","id":"Row 889"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:878"}},"data":"Data for row 89","id":"Row 89"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:879"}},"data":"Data for row 890","id":"Row 890"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:880"}},"data":"Data for row 891","id":"Row 891"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:881"}},"data":"Data for row 892","id":"Row 892"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:882"}},"data":"Data for row 893","id":"Row 893"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:883"}},"data":"Data for row 894","id":"Row 894"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:884"}},"data":"Data for row 895","id":"Row 895"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:885"}},"data":"Data for row 896","id":"Row 896"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:886"}},"data":"Data for row 897","id":"Row 897"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:887"}},"data":"Data for row 898","id":"Row 898"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:888"}},"data":"Data for row 899","id":"Row 899"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:889"}},"data":"Data for row 9","id":"Row 9"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:890"}},"data":"Data for row 90","id":"Row 90"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:891"}},"data":"Data for row 900","id":"Row 900"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:892"}},"data":"Data for row 901","id":"Row 901"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:893"}},"data":"Data for row 902","id":"Row 902"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:894"}},"data":"Data for row 903","id":"Row 903"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:895"}},"data":"Data for row 904","id":"Row 904"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:896"}},"data":"Data for row 905","id":"Row 905"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:897"}},"data":"Data for row 906","id":"Row 906"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:898"}},"data":"Data for row 907","id":"Row 907"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:899"}},"data":"Data for row 908","id":"Row 908"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:900"}},"data":"Data for row 909","id":"Row 909"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:901"}},"data":"Data for row 91","id":"Row 91"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:902"}},"data":"Data for row 910","id":"Row 910"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:903"}},"data":"Data for row 911","id":"Row 911"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:904"}},"data":"Data for row 912","id":"Row 912"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:905"}},"data":"Data for row 913","id":"Row 913"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:906"}},"data":"Data for row 914","id":"Row 914"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:907"}},"data":"Data for row 915","id":"Row 915"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:908"}},"data":"Data for row 916","id":"Row 916"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:909"}},"data":"Data for row 917","id":"Row 917"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:910"}},"data":"Data for row 918","id":"Row 918"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:911"}},"data":"Data for row 919","id":"Row 919"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:912"}},"data":"Data for row 92","id":"Row 92"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:913"}},"data":"Data for row 920","id":"Row 920"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:914"}},"data":"Data for row 921","id":"Row 921"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:915"}},"data":"Data for row 922","id":"Row 922"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:916"}},"data":"Data for row 923","id":"Row 923"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:917"}},"data":"Data for row 924","id":"Row 924"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:918"}},"data":"Data for row 925","id":"Row 925"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:919"}},"data":"Data for row 926","id":"Row 926"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:920"}},"data":"Data for row 927","id":"Row 927"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:921"}},"data":"Data for row 928","id":"Row 928"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:922"}},"data":"Data for row 929","id":"Row 929"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:923"}},"data":"Data for row 93","id":"Row 93"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:924"}},"data":"Data for row 930","id":"Row 930"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:925"}},"data":"Data for row 931","id":"Row 931"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:926"}},"data":"Data for row 932","id":"Row 932"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:927"}},"data":"Data for row 933","id":"Row 933"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:928"}},"data":"Data for row 934","id":"Row 934"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:929"}},"data":"Data for row 935","id":"Row 935"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:930"}},"data":"Data for row 936","id":"Row 936"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:931"}},"data":"Data for row 937","id":"Row 937"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:932"}},"data":"Data for row 938","id":"Row 938"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:933"}},"data":"Data for row 939","id":"Row 939"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:934"}},"data":"Data for row 94","id":"Row 94"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:935"}},"data":"Data for row 940","id":"Row 940"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:936"}},"data":"Data for row 941","id":"Row 941"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:937"}},"data":"Data for row 942","id":"Row 942"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:938"}},"data":"Data for row 943","id":"Row 943"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:939"}},"data":"Data for row 944","id":"Row 944"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:940"}},"data":"Data for row 945","id":"Row 945"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:941"}},"data":"Data for row 946","id":"Row 946"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:942"}},"data":"Data for row 947","id":"Row 947"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:943"}},"data":"Data for row 948","id":"Row 948"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:944"}},"data":"Data for row 949","id":"Row 949"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:945"}},"data":"Data for row 95","id":"Row 95"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:946"}},"data":"Data for row 950","id":"Row 950"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:947"}},"data":"Data for row 951","id":"Row 951"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:948"}},"data":"Data for row 952","id":"Row 952"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:949"}},"data":"Data for row 953","id":"Row 953"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:950"}},"data":"Data for row 954","id":"Row 954"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:951"}},"data":"Data for row 955","id":"Row 955"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:952"}},"data":"Data for row 956","id":"Row 956"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:953"}},"data":"Data for row 957","id":"Row 957"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:954"}},"data":"Data for row 958","id":"Row 958"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:955"}},"data":"Data for row 959","id":"Row 959"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:956"}},"data":"Data for row 96","id":"Row 96"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:957"}},"data":"Data for row 960","id":"Row 960"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:958"}},"data":"Data for row 961","id":"Row 961"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:959"}},"data":"Data for row 962","id":"Row 962"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:960"}},"data":"Data for row 963","id":"Row 963"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:961"}},"data":"Data for row 964","id":"Row 964"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:962"}},"data":"Data for row 965","id":"Row 965"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:963"}},"data":"Data for row 966","id":"Row 966"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:964"}},"data":"Data for row 967","id":"Row 967"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:965"}},"data":"Data for row 968","id":"Row 968"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:966"}},"data":"Data for row 969","id":"Row 969"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:967"}},"data":"Data for row 97","id":"Row 97"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:968"}},"data":"Data for row 970","id":"Row 970"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:969"}},"data":"Data for row 971","id":"Row 971"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:970"}},"data":"Data for row 972","id":"Row 972"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:971"}},"data":"Data for row 973","id":"Row 973"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:972"}},"data":"Data for row 974","id":"Row 974"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:973"}},"data":"Data for row 975","id":"Row 975"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:974"}},"data":"Data for row 976","id":"Row 976"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:975"}},"data":"Data for row 977","id":"Row 977"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:976"}},"data":"Data for row 978","id":"Row 978"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:977"}},"data":"Data for row 979","id":"Row 979"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:978"}},"data":"Data for row 98","id":"Row 98"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:979"}},"data":"Data for row 980","id":"Row 980"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:980"}},"data":"Data for row 981","id":"Row 981"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:981"}},"data":"Data for row 982","id":"Row 982"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:982"}},"data":"Data for row 983","id":"Row 983"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:983"}},"data":"Data for row 984","id":"Row 984"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:984"}},"data":"Data for row 985","id":"Row 985"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:985"}},"data":"Data for row 986","id":"Row 986"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:986"}},"data":"Data for row 987","id":"Row 987"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:987"}},"data":"Data for row 988","id":"Row 988"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:988"}},"data":"Data for row 989","id":"Row 989"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:989"}},"data":"Data for row 99","id":"Row 99"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:990"}},"data":"Data for row 990","id":"Row 990"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:991"}},"data":"Data for row 991","id":"Row 991"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:992"}},"data":"Data for row 992","id":"Row 992"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:993"}},"data":"Data for row 993","id":"Row 993"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:994"}},"data":"Data for row 994","id":"Row 994"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:995"}},"data":"Data for row 995","id":"Row 995"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:996"}},"data":"Data for row 996","id":"Row 996"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:997"}},"data":"Data for row 997","id":"Row 997"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:998"}},"data":"Data for row 998","id":"Row 998"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_11837744","cursor":"backfill:999"}},"data":"Data for row 999","id":"Row 999"} +# ================================ +# Collection "acmeCo/test/test_backfillmodes_25282936": 1000 Documents +# ================================ +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:0"}},"data":"Data for row 0","id":"Row 0"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:1"}},"data":"Data for row 1","id":"Row 1"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:2"}},"data":"Data for row 10","id":"Row 10"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:3"}},"data":"Data for row 100","id":"Row 100"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:4"}},"data":"Data for row 101","id":"Row 101"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:5"}},"data":"Data for row 102","id":"Row 102"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:6"}},"data":"Data for row 103","id":"Row 103"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:7"}},"data":"Data for row 104","id":"Row 104"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:8"}},"data":"Data for row 105","id":"Row 105"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:9"}},"data":"Data for row 106","id":"Row 106"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:10"}},"data":"Data for row 107","id":"Row 107"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:11"}},"data":"Data for row 108","id":"Row 108"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:12"}},"data":"Data for row 109","id":"Row 109"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:13"}},"data":"Data for row 11","id":"Row 11"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:14"}},"data":"Data for row 110","id":"Row 110"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:15"}},"data":"Data for row 111","id":"Row 111"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:16"}},"data":"Data for row 112","id":"Row 112"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:17"}},"data":"Data for row 113","id":"Row 113"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:18"}},"data":"Data for row 114","id":"Row 114"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:19"}},"data":"Data for row 115","id":"Row 115"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:20"}},"data":"Data for row 116","id":"Row 116"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:21"}},"data":"Data for row 117","id":"Row 117"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:22"}},"data":"Data for row 118","id":"Row 118"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:23"}},"data":"Data for row 119","id":"Row 119"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:24"}},"data":"Data for row 12","id":"Row 12"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:25"}},"data":"Data for row 120","id":"Row 120"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:26"}},"data":"Data for row 121","id":"Row 121"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:27"}},"data":"Data for row 122","id":"Row 122"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:28"}},"data":"Data for row 123","id":"Row 123"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:29"}},"data":"Data for row 124","id":"Row 124"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:30"}},"data":"Data for row 125","id":"Row 125"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:31"}},"data":"Data for row 126","id":"Row 126"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:32"}},"data":"Data for row 127","id":"Row 127"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:33"}},"data":"Data for row 128","id":"Row 128"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:34"}},"data":"Data for row 129","id":"Row 129"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:35"}},"data":"Data for row 13","id":"Row 13"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:36"}},"data":"Data for row 130","id":"Row 130"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:37"}},"data":"Data for row 131","id":"Row 131"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:38"}},"data":"Data for row 132","id":"Row 132"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:39"}},"data":"Data for row 133","id":"Row 133"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:40"}},"data":"Data for row 134","id":"Row 134"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:41"}},"data":"Data for row 135","id":"Row 135"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:42"}},"data":"Data for row 136","id":"Row 136"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:43"}},"data":"Data for row 137","id":"Row 137"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:44"}},"data":"Data for row 138","id":"Row 138"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:45"}},"data":"Data for row 139","id":"Row 139"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:46"}},"data":"Data for row 14","id":"Row 14"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:47"}},"data":"Data for row 140","id":"Row 140"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:48"}},"data":"Data for row 141","id":"Row 141"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:49"}},"data":"Data for row 142","id":"Row 142"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:50"}},"data":"Data for row 143","id":"Row 143"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:51"}},"data":"Data for row 144","id":"Row 144"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:52"}},"data":"Data for row 145","id":"Row 145"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:53"}},"data":"Data for row 146","id":"Row 146"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:54"}},"data":"Data for row 147","id":"Row 147"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:55"}},"data":"Data for row 148","id":"Row 148"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:56"}},"data":"Data for row 149","id":"Row 149"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:57"}},"data":"Data for row 15","id":"Row 15"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:58"}},"data":"Data for row 150","id":"Row 150"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:59"}},"data":"Data for row 151","id":"Row 151"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:60"}},"data":"Data for row 152","id":"Row 152"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:61"}},"data":"Data for row 153","id":"Row 153"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:62"}},"data":"Data for row 154","id":"Row 154"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:63"}},"data":"Data for row 155","id":"Row 155"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:64"}},"data":"Data for row 156","id":"Row 156"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:65"}},"data":"Data for row 157","id":"Row 157"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:66"}},"data":"Data for row 158","id":"Row 158"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:67"}},"data":"Data for row 159","id":"Row 159"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:68"}},"data":"Data for row 16","id":"Row 16"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:69"}},"data":"Data for row 160","id":"Row 160"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:70"}},"data":"Data for row 161","id":"Row 161"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:71"}},"data":"Data for row 162","id":"Row 162"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:72"}},"data":"Data for row 163","id":"Row 163"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:73"}},"data":"Data for row 164","id":"Row 164"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:74"}},"data":"Data for row 165","id":"Row 165"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:75"}},"data":"Data for row 166","id":"Row 166"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:76"}},"data":"Data for row 167","id":"Row 167"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:77"}},"data":"Data for row 168","id":"Row 168"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:78"}},"data":"Data for row 169","id":"Row 169"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:79"}},"data":"Data for row 17","id":"Row 17"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:80"}},"data":"Data for row 170","id":"Row 170"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:81"}},"data":"Data for row 171","id":"Row 171"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:82"}},"data":"Data for row 172","id":"Row 172"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:83"}},"data":"Data for row 173","id":"Row 173"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:84"}},"data":"Data for row 174","id":"Row 174"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:85"}},"data":"Data for row 175","id":"Row 175"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:86"}},"data":"Data for row 176","id":"Row 176"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:87"}},"data":"Data for row 177","id":"Row 177"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:88"}},"data":"Data for row 178","id":"Row 178"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:89"}},"data":"Data for row 179","id":"Row 179"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:90"}},"data":"Data for row 18","id":"Row 18"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:91"}},"data":"Data for row 180","id":"Row 180"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:92"}},"data":"Data for row 181","id":"Row 181"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:93"}},"data":"Data for row 182","id":"Row 182"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:94"}},"data":"Data for row 183","id":"Row 183"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:95"}},"data":"Data for row 184","id":"Row 184"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:96"}},"data":"Data for row 185","id":"Row 185"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:97"}},"data":"Data for row 186","id":"Row 186"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:98"}},"data":"Data for row 187","id":"Row 187"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:99"}},"data":"Data for row 188","id":"Row 188"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:100"}},"data":"Data for row 189","id":"Row 189"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:101"}},"data":"Data for row 19","id":"Row 19"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:102"}},"data":"Data for row 190","id":"Row 190"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:103"}},"data":"Data for row 191","id":"Row 191"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:104"}},"data":"Data for row 192","id":"Row 192"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:105"}},"data":"Data for row 193","id":"Row 193"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:106"}},"data":"Data for row 194","id":"Row 194"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:107"}},"data":"Data for row 195","id":"Row 195"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:108"}},"data":"Data for row 196","id":"Row 196"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:109"}},"data":"Data for row 197","id":"Row 197"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:110"}},"data":"Data for row 198","id":"Row 198"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:111"}},"data":"Data for row 199","id":"Row 199"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:112"}},"data":"Data for row 2","id":"Row 2"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:113"}},"data":"Data for row 20","id":"Row 20"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:114"}},"data":"Data for row 200","id":"Row 200"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:115"}},"data":"Data for row 201","id":"Row 201"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:116"}},"data":"Data for row 202","id":"Row 202"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:117"}},"data":"Data for row 203","id":"Row 203"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:118"}},"data":"Data for row 204","id":"Row 204"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:119"}},"data":"Data for row 205","id":"Row 205"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:120"}},"data":"Data for row 206","id":"Row 206"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:121"}},"data":"Data for row 207","id":"Row 207"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:122"}},"data":"Data for row 208","id":"Row 208"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:123"}},"data":"Data for row 209","id":"Row 209"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:124"}},"data":"Data for row 21","id":"Row 21"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:125"}},"data":"Data for row 210","id":"Row 210"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:126"}},"data":"Data for row 211","id":"Row 211"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:127"}},"data":"Data for row 212","id":"Row 212"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:128"}},"data":"Data for row 213","id":"Row 213"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:129"}},"data":"Data for row 214","id":"Row 214"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:130"}},"data":"Data for row 215","id":"Row 215"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:131"}},"data":"Data for row 216","id":"Row 216"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:132"}},"data":"Data for row 217","id":"Row 217"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:133"}},"data":"Data for row 218","id":"Row 218"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:134"}},"data":"Data for row 219","id":"Row 219"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:135"}},"data":"Data for row 22","id":"Row 22"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:136"}},"data":"Data for row 220","id":"Row 220"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:137"}},"data":"Data for row 221","id":"Row 221"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:138"}},"data":"Data for row 222","id":"Row 222"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:139"}},"data":"Data for row 223","id":"Row 223"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:140"}},"data":"Data for row 224","id":"Row 224"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:141"}},"data":"Data for row 225","id":"Row 225"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:142"}},"data":"Data for row 226","id":"Row 226"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:143"}},"data":"Data for row 227","id":"Row 227"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:144"}},"data":"Data for row 228","id":"Row 228"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:145"}},"data":"Data for row 229","id":"Row 229"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:146"}},"data":"Data for row 23","id":"Row 23"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:147"}},"data":"Data for row 230","id":"Row 230"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:148"}},"data":"Data for row 231","id":"Row 231"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:149"}},"data":"Data for row 232","id":"Row 232"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:150"}},"data":"Data for row 233","id":"Row 233"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:151"}},"data":"Data for row 234","id":"Row 234"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:152"}},"data":"Data for row 235","id":"Row 235"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:153"}},"data":"Data for row 236","id":"Row 236"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:154"}},"data":"Data for row 237","id":"Row 237"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:155"}},"data":"Data for row 238","id":"Row 238"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:156"}},"data":"Data for row 239","id":"Row 239"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:157"}},"data":"Data for row 24","id":"Row 24"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:158"}},"data":"Data for row 240","id":"Row 240"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:159"}},"data":"Data for row 241","id":"Row 241"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:160"}},"data":"Data for row 242","id":"Row 242"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:161"}},"data":"Data for row 243","id":"Row 243"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:162"}},"data":"Data for row 244","id":"Row 244"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:163"}},"data":"Data for row 245","id":"Row 245"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:164"}},"data":"Data for row 246","id":"Row 246"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:165"}},"data":"Data for row 247","id":"Row 247"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:166"}},"data":"Data for row 248","id":"Row 248"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:167"}},"data":"Data for row 249","id":"Row 249"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:168"}},"data":"Data for row 25","id":"Row 25"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:169"}},"data":"Data for row 250","id":"Row 250"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:170"}},"data":"Data for row 251","id":"Row 251"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:171"}},"data":"Data for row 252","id":"Row 252"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:172"}},"data":"Data for row 253","id":"Row 253"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:173"}},"data":"Data for row 254","id":"Row 254"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:174"}},"data":"Data for row 255","id":"Row 255"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:175"}},"data":"Data for row 256","id":"Row 256"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:176"}},"data":"Data for row 257","id":"Row 257"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:177"}},"data":"Data for row 258","id":"Row 258"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:178"}},"data":"Data for row 259","id":"Row 259"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:179"}},"data":"Data for row 26","id":"Row 26"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:180"}},"data":"Data for row 260","id":"Row 260"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:181"}},"data":"Data for row 261","id":"Row 261"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:182"}},"data":"Data for row 262","id":"Row 262"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:183"}},"data":"Data for row 263","id":"Row 263"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:184"}},"data":"Data for row 264","id":"Row 264"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:185"}},"data":"Data for row 265","id":"Row 265"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:186"}},"data":"Data for row 266","id":"Row 266"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:187"}},"data":"Data for row 267","id":"Row 267"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:188"}},"data":"Data for row 268","id":"Row 268"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:189"}},"data":"Data for row 269","id":"Row 269"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:190"}},"data":"Data for row 27","id":"Row 27"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:191"}},"data":"Data for row 270","id":"Row 270"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:192"}},"data":"Data for row 271","id":"Row 271"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:193"}},"data":"Data for row 272","id":"Row 272"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:194"}},"data":"Data for row 273","id":"Row 273"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:195"}},"data":"Data for row 274","id":"Row 274"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:196"}},"data":"Data for row 275","id":"Row 275"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:197"}},"data":"Data for row 276","id":"Row 276"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:198"}},"data":"Data for row 277","id":"Row 277"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:199"}},"data":"Data for row 278","id":"Row 278"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:200"}},"data":"Data for row 279","id":"Row 279"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:201"}},"data":"Data for row 28","id":"Row 28"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:202"}},"data":"Data for row 280","id":"Row 280"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:203"}},"data":"Data for row 281","id":"Row 281"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:204"}},"data":"Data for row 282","id":"Row 282"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:205"}},"data":"Data for row 283","id":"Row 283"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:206"}},"data":"Data for row 284","id":"Row 284"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:207"}},"data":"Data for row 285","id":"Row 285"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:208"}},"data":"Data for row 286","id":"Row 286"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:209"}},"data":"Data for row 287","id":"Row 287"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:210"}},"data":"Data for row 288","id":"Row 288"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:211"}},"data":"Data for row 289","id":"Row 289"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:212"}},"data":"Data for row 29","id":"Row 29"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:213"}},"data":"Data for row 290","id":"Row 290"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:214"}},"data":"Data for row 291","id":"Row 291"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:215"}},"data":"Data for row 292","id":"Row 292"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:216"}},"data":"Data for row 293","id":"Row 293"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:217"}},"data":"Data for row 294","id":"Row 294"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:218"}},"data":"Data for row 295","id":"Row 295"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:219"}},"data":"Data for row 296","id":"Row 296"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:220"}},"data":"Data for row 297","id":"Row 297"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:221"}},"data":"Data for row 298","id":"Row 298"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:222"}},"data":"Data for row 299","id":"Row 299"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:223"}},"data":"Data for row 3","id":"Row 3"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:224"}},"data":"Data for row 30","id":"Row 30"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:225"}},"data":"Data for row 300","id":"Row 300"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:226"}},"data":"Data for row 301","id":"Row 301"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:227"}},"data":"Data for row 302","id":"Row 302"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:228"}},"data":"Data for row 303","id":"Row 303"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:229"}},"data":"Data for row 304","id":"Row 304"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:230"}},"data":"Data for row 305","id":"Row 305"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:231"}},"data":"Data for row 306","id":"Row 306"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:232"}},"data":"Data for row 307","id":"Row 307"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:233"}},"data":"Data for row 308","id":"Row 308"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:234"}},"data":"Data for row 309","id":"Row 309"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:235"}},"data":"Data for row 31","id":"Row 31"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:236"}},"data":"Data for row 310","id":"Row 310"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:237"}},"data":"Data for row 311","id":"Row 311"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:238"}},"data":"Data for row 312","id":"Row 312"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:239"}},"data":"Data for row 313","id":"Row 313"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:240"}},"data":"Data for row 314","id":"Row 314"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:241"}},"data":"Data for row 315","id":"Row 315"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:242"}},"data":"Data for row 316","id":"Row 316"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:243"}},"data":"Data for row 317","id":"Row 317"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:244"}},"data":"Data for row 318","id":"Row 318"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:245"}},"data":"Data for row 319","id":"Row 319"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:246"}},"data":"Data for row 32","id":"Row 32"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:247"}},"data":"Data for row 320","id":"Row 320"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:248"}},"data":"Data for row 321","id":"Row 321"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:249"}},"data":"Data for row 322","id":"Row 322"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:250"}},"data":"Data for row 323","id":"Row 323"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:251"}},"data":"Data for row 324","id":"Row 324"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:252"}},"data":"Data for row 325","id":"Row 325"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:253"}},"data":"Data for row 326","id":"Row 326"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:254"}},"data":"Data for row 327","id":"Row 327"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:255"}},"data":"Data for row 328","id":"Row 328"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:256"}},"data":"Data for row 329","id":"Row 329"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:257"}},"data":"Data for row 33","id":"Row 33"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:258"}},"data":"Data for row 330","id":"Row 330"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:259"}},"data":"Data for row 331","id":"Row 331"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:260"}},"data":"Data for row 332","id":"Row 332"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:261"}},"data":"Data for row 333","id":"Row 333"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:262"}},"data":"Data for row 334","id":"Row 334"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:263"}},"data":"Data for row 335","id":"Row 335"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:264"}},"data":"Data for row 336","id":"Row 336"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:265"}},"data":"Data for row 337","id":"Row 337"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:266"}},"data":"Data for row 338","id":"Row 338"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:267"}},"data":"Data for row 339","id":"Row 339"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:268"}},"data":"Data for row 34","id":"Row 34"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:269"}},"data":"Data for row 340","id":"Row 340"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:270"}},"data":"Data for row 341","id":"Row 341"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:271"}},"data":"Data for row 342","id":"Row 342"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:272"}},"data":"Data for row 343","id":"Row 343"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:273"}},"data":"Data for row 344","id":"Row 344"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:274"}},"data":"Data for row 345","id":"Row 345"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:275"}},"data":"Data for row 346","id":"Row 346"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:276"}},"data":"Data for row 347","id":"Row 347"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:277"}},"data":"Data for row 348","id":"Row 348"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:278"}},"data":"Data for row 349","id":"Row 349"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:279"}},"data":"Data for row 35","id":"Row 35"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:280"}},"data":"Data for row 350","id":"Row 350"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:281"}},"data":"Data for row 351","id":"Row 351"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:282"}},"data":"Data for row 352","id":"Row 352"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:283"}},"data":"Data for row 353","id":"Row 353"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:284"}},"data":"Data for row 354","id":"Row 354"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:285"}},"data":"Data for row 355","id":"Row 355"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:286"}},"data":"Data for row 356","id":"Row 356"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:287"}},"data":"Data for row 357","id":"Row 357"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:288"}},"data":"Data for row 358","id":"Row 358"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:289"}},"data":"Data for row 359","id":"Row 359"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:290"}},"data":"Data for row 36","id":"Row 36"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:291"}},"data":"Data for row 360","id":"Row 360"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:292"}},"data":"Data for row 361","id":"Row 361"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:293"}},"data":"Data for row 362","id":"Row 362"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:294"}},"data":"Data for row 363","id":"Row 363"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:295"}},"data":"Data for row 364","id":"Row 364"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:296"}},"data":"Data for row 365","id":"Row 365"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:297"}},"data":"Data for row 366","id":"Row 366"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:298"}},"data":"Data for row 367","id":"Row 367"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:299"}},"data":"Data for row 368","id":"Row 368"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:300"}},"data":"Data for row 369","id":"Row 369"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:301"}},"data":"Data for row 37","id":"Row 37"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:302"}},"data":"Data for row 370","id":"Row 370"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:303"}},"data":"Data for row 371","id":"Row 371"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:304"}},"data":"Data for row 372","id":"Row 372"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:305"}},"data":"Data for row 373","id":"Row 373"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:306"}},"data":"Data for row 374","id":"Row 374"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:307"}},"data":"Data for row 375","id":"Row 375"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:308"}},"data":"Data for row 376","id":"Row 376"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:309"}},"data":"Data for row 377","id":"Row 377"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:310"}},"data":"Data for row 378","id":"Row 378"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:311"}},"data":"Data for row 379","id":"Row 379"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:312"}},"data":"Data for row 38","id":"Row 38"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:313"}},"data":"Data for row 380","id":"Row 380"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:314"}},"data":"Data for row 381","id":"Row 381"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:315"}},"data":"Data for row 382","id":"Row 382"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:316"}},"data":"Data for row 383","id":"Row 383"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:317"}},"data":"Data for row 384","id":"Row 384"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:318"}},"data":"Data for row 385","id":"Row 385"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:319"}},"data":"Data for row 386","id":"Row 386"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:320"}},"data":"Data for row 387","id":"Row 387"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:321"}},"data":"Data for row 388","id":"Row 388"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:322"}},"data":"Data for row 389","id":"Row 389"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:323"}},"data":"Data for row 39","id":"Row 39"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:324"}},"data":"Data for row 390","id":"Row 390"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:325"}},"data":"Data for row 391","id":"Row 391"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:326"}},"data":"Data for row 392","id":"Row 392"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:327"}},"data":"Data for row 393","id":"Row 393"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:328"}},"data":"Data for row 394","id":"Row 394"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:329"}},"data":"Data for row 395","id":"Row 395"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:330"}},"data":"Data for row 396","id":"Row 396"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:331"}},"data":"Data for row 397","id":"Row 397"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:332"}},"data":"Data for row 398","id":"Row 398"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:333"}},"data":"Data for row 399","id":"Row 399"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:334"}},"data":"Data for row 4","id":"Row 4"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:335"}},"data":"Data for row 40","id":"Row 40"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:336"}},"data":"Data for row 400","id":"Row 400"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:337"}},"data":"Data for row 401","id":"Row 401"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:338"}},"data":"Data for row 402","id":"Row 402"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:339"}},"data":"Data for row 403","id":"Row 403"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:340"}},"data":"Data for row 404","id":"Row 404"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:341"}},"data":"Data for row 405","id":"Row 405"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:342"}},"data":"Data for row 406","id":"Row 406"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:343"}},"data":"Data for row 407","id":"Row 407"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:344"}},"data":"Data for row 408","id":"Row 408"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:345"}},"data":"Data for row 409","id":"Row 409"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:346"}},"data":"Data for row 41","id":"Row 41"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:347"}},"data":"Data for row 410","id":"Row 410"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:348"}},"data":"Data for row 411","id":"Row 411"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:349"}},"data":"Data for row 412","id":"Row 412"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:350"}},"data":"Data for row 413","id":"Row 413"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:351"}},"data":"Data for row 414","id":"Row 414"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:352"}},"data":"Data for row 415","id":"Row 415"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:353"}},"data":"Data for row 416","id":"Row 416"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:354"}},"data":"Data for row 417","id":"Row 417"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:355"}},"data":"Data for row 418","id":"Row 418"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:356"}},"data":"Data for row 419","id":"Row 419"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:357"}},"data":"Data for row 42","id":"Row 42"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:358"}},"data":"Data for row 420","id":"Row 420"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:359"}},"data":"Data for row 421","id":"Row 421"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:360"}},"data":"Data for row 422","id":"Row 422"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:361"}},"data":"Data for row 423","id":"Row 423"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:362"}},"data":"Data for row 424","id":"Row 424"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:363"}},"data":"Data for row 425","id":"Row 425"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:364"}},"data":"Data for row 426","id":"Row 426"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:365"}},"data":"Data for row 427","id":"Row 427"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:366"}},"data":"Data for row 428","id":"Row 428"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:367"}},"data":"Data for row 429","id":"Row 429"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:368"}},"data":"Data for row 43","id":"Row 43"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:369"}},"data":"Data for row 430","id":"Row 430"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:370"}},"data":"Data for row 431","id":"Row 431"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:371"}},"data":"Data for row 432","id":"Row 432"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:372"}},"data":"Data for row 433","id":"Row 433"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:373"}},"data":"Data for row 434","id":"Row 434"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:374"}},"data":"Data for row 435","id":"Row 435"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:375"}},"data":"Data for row 436","id":"Row 436"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:376"}},"data":"Data for row 437","id":"Row 437"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:377"}},"data":"Data for row 438","id":"Row 438"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:378"}},"data":"Data for row 439","id":"Row 439"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:379"}},"data":"Data for row 44","id":"Row 44"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:380"}},"data":"Data for row 440","id":"Row 440"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:381"}},"data":"Data for row 441","id":"Row 441"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:382"}},"data":"Data for row 442","id":"Row 442"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:383"}},"data":"Data for row 443","id":"Row 443"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:384"}},"data":"Data for row 444","id":"Row 444"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:385"}},"data":"Data for row 445","id":"Row 445"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:386"}},"data":"Data for row 446","id":"Row 446"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:387"}},"data":"Data for row 447","id":"Row 447"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:388"}},"data":"Data for row 448","id":"Row 448"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:389"}},"data":"Data for row 449","id":"Row 449"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:390"}},"data":"Data for row 45","id":"Row 45"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:391"}},"data":"Data for row 450","id":"Row 450"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:392"}},"data":"Data for row 451","id":"Row 451"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:393"}},"data":"Data for row 452","id":"Row 452"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:394"}},"data":"Data for row 453","id":"Row 453"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:395"}},"data":"Data for row 454","id":"Row 454"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:396"}},"data":"Data for row 455","id":"Row 455"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:397"}},"data":"Data for row 456","id":"Row 456"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:398"}},"data":"Data for row 457","id":"Row 457"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:399"}},"data":"Data for row 458","id":"Row 458"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:400"}},"data":"Data for row 459","id":"Row 459"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:401"}},"data":"Data for row 46","id":"Row 46"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:402"}},"data":"Data for row 460","id":"Row 460"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:403"}},"data":"Data for row 461","id":"Row 461"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:404"}},"data":"Data for row 462","id":"Row 462"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:405"}},"data":"Data for row 463","id":"Row 463"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:406"}},"data":"Data for row 464","id":"Row 464"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:407"}},"data":"Data for row 465","id":"Row 465"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:408"}},"data":"Data for row 466","id":"Row 466"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:409"}},"data":"Data for row 467","id":"Row 467"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:410"}},"data":"Data for row 468","id":"Row 468"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:411"}},"data":"Data for row 469","id":"Row 469"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:412"}},"data":"Data for row 47","id":"Row 47"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:413"}},"data":"Data for row 470","id":"Row 470"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:414"}},"data":"Data for row 471","id":"Row 471"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:415"}},"data":"Data for row 472","id":"Row 472"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:416"}},"data":"Data for row 473","id":"Row 473"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:417"}},"data":"Data for row 474","id":"Row 474"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:418"}},"data":"Data for row 475","id":"Row 475"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:419"}},"data":"Data for row 476","id":"Row 476"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:420"}},"data":"Data for row 477","id":"Row 477"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:421"}},"data":"Data for row 478","id":"Row 478"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:422"}},"data":"Data for row 479","id":"Row 479"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:423"}},"data":"Data for row 48","id":"Row 48"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:424"}},"data":"Data for row 480","id":"Row 480"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:425"}},"data":"Data for row 481","id":"Row 481"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:426"}},"data":"Data for row 482","id":"Row 482"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:427"}},"data":"Data for row 483","id":"Row 483"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:428"}},"data":"Data for row 484","id":"Row 484"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:429"}},"data":"Data for row 485","id":"Row 485"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:430"}},"data":"Data for row 486","id":"Row 486"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:431"}},"data":"Data for row 487","id":"Row 487"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:432"}},"data":"Data for row 488","id":"Row 488"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:433"}},"data":"Data for row 489","id":"Row 489"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:434"}},"data":"Data for row 49","id":"Row 49"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:435"}},"data":"Data for row 490","id":"Row 490"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:436"}},"data":"Data for row 491","id":"Row 491"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:437"}},"data":"Data for row 492","id":"Row 492"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:438"}},"data":"Data for row 493","id":"Row 493"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:439"}},"data":"Data for row 494","id":"Row 494"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:440"}},"data":"Data for row 495","id":"Row 495"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:441"}},"data":"Data for row 496","id":"Row 496"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:442"}},"data":"Data for row 497","id":"Row 497"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:443"}},"data":"Data for row 498","id":"Row 498"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:444"}},"data":"Data for row 499","id":"Row 499"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:445"}},"data":"Data for row 5","id":"Row 5"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:446"}},"data":"Data for row 50","id":"Row 50"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:447"}},"data":"Data for row 500","id":"Row 500"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:448"}},"data":"Data for row 501","id":"Row 501"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:449"}},"data":"Data for row 502","id":"Row 502"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:450"}},"data":"Data for row 503","id":"Row 503"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:451"}},"data":"Data for row 504","id":"Row 504"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:452"}},"data":"Data for row 505","id":"Row 505"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:453"}},"data":"Data for row 506","id":"Row 506"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:454"}},"data":"Data for row 507","id":"Row 507"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:455"}},"data":"Data for row 508","id":"Row 508"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:456"}},"data":"Data for row 509","id":"Row 509"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:457"}},"data":"Data for row 51","id":"Row 51"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:458"}},"data":"Data for row 510","id":"Row 510"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:459"}},"data":"Data for row 511","id":"Row 511"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:460"}},"data":"Data for row 512","id":"Row 512"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:461"}},"data":"Data for row 513","id":"Row 513"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:462"}},"data":"Data for row 514","id":"Row 514"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:463"}},"data":"Data for row 515","id":"Row 515"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:464"}},"data":"Data for row 516","id":"Row 516"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:465"}},"data":"Data for row 517","id":"Row 517"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:466"}},"data":"Data for row 518","id":"Row 518"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:467"}},"data":"Data for row 519","id":"Row 519"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:468"}},"data":"Data for row 52","id":"Row 52"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:469"}},"data":"Data for row 520","id":"Row 520"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:470"}},"data":"Data for row 521","id":"Row 521"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:471"}},"data":"Data for row 522","id":"Row 522"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:472"}},"data":"Data for row 523","id":"Row 523"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:473"}},"data":"Data for row 524","id":"Row 524"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:474"}},"data":"Data for row 525","id":"Row 525"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:475"}},"data":"Data for row 526","id":"Row 526"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:476"}},"data":"Data for row 527","id":"Row 527"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:477"}},"data":"Data for row 528","id":"Row 528"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:478"}},"data":"Data for row 529","id":"Row 529"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:479"}},"data":"Data for row 53","id":"Row 53"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:480"}},"data":"Data for row 530","id":"Row 530"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:481"}},"data":"Data for row 531","id":"Row 531"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:482"}},"data":"Data for row 532","id":"Row 532"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:483"}},"data":"Data for row 533","id":"Row 533"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:484"}},"data":"Data for row 534","id":"Row 534"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:485"}},"data":"Data for row 535","id":"Row 535"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:486"}},"data":"Data for row 536","id":"Row 536"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:487"}},"data":"Data for row 537","id":"Row 537"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:488"}},"data":"Data for row 538","id":"Row 538"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:489"}},"data":"Data for row 539","id":"Row 539"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:490"}},"data":"Data for row 54","id":"Row 54"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:491"}},"data":"Data for row 540","id":"Row 540"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:492"}},"data":"Data for row 541","id":"Row 541"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:493"}},"data":"Data for row 542","id":"Row 542"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:494"}},"data":"Data for row 543","id":"Row 543"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:495"}},"data":"Data for row 544","id":"Row 544"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:496"}},"data":"Data for row 545","id":"Row 545"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:497"}},"data":"Data for row 546","id":"Row 546"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:498"}},"data":"Data for row 547","id":"Row 547"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:499"}},"data":"Data for row 548","id":"Row 548"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:500"}},"data":"Data for row 549","id":"Row 549"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:501"}},"data":"Data for row 55","id":"Row 55"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:502"}},"data":"Data for row 550","id":"Row 550"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:503"}},"data":"Data for row 551","id":"Row 551"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:504"}},"data":"Data for row 552","id":"Row 552"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:505"}},"data":"Data for row 553","id":"Row 553"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:506"}},"data":"Data for row 554","id":"Row 554"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:507"}},"data":"Data for row 555","id":"Row 555"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:508"}},"data":"Data for row 556","id":"Row 556"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:509"}},"data":"Data for row 557","id":"Row 557"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:510"}},"data":"Data for row 558","id":"Row 558"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:511"}},"data":"Data for row 559","id":"Row 559"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:512"}},"data":"Data for row 56","id":"Row 56"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:513"}},"data":"Data for row 560","id":"Row 560"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:514"}},"data":"Data for row 561","id":"Row 561"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:515"}},"data":"Data for row 562","id":"Row 562"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:516"}},"data":"Data for row 563","id":"Row 563"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:517"}},"data":"Data for row 564","id":"Row 564"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:518"}},"data":"Data for row 565","id":"Row 565"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:519"}},"data":"Data for row 566","id":"Row 566"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:520"}},"data":"Data for row 567","id":"Row 567"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:521"}},"data":"Data for row 568","id":"Row 568"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:522"}},"data":"Data for row 569","id":"Row 569"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:523"}},"data":"Data for row 57","id":"Row 57"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:524"}},"data":"Data for row 570","id":"Row 570"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:525"}},"data":"Data for row 571","id":"Row 571"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:526"}},"data":"Data for row 572","id":"Row 572"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:527"}},"data":"Data for row 573","id":"Row 573"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:528"}},"data":"Data for row 574","id":"Row 574"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:529"}},"data":"Data for row 575","id":"Row 575"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:530"}},"data":"Data for row 576","id":"Row 576"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:531"}},"data":"Data for row 577","id":"Row 577"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:532"}},"data":"Data for row 578","id":"Row 578"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:533"}},"data":"Data for row 579","id":"Row 579"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:534"}},"data":"Data for row 58","id":"Row 58"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:535"}},"data":"Data for row 580","id":"Row 580"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:536"}},"data":"Data for row 581","id":"Row 581"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:537"}},"data":"Data for row 582","id":"Row 582"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:538"}},"data":"Data for row 583","id":"Row 583"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:539"}},"data":"Data for row 584","id":"Row 584"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:540"}},"data":"Data for row 585","id":"Row 585"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:541"}},"data":"Data for row 586","id":"Row 586"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:542"}},"data":"Data for row 587","id":"Row 587"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:543"}},"data":"Data for row 588","id":"Row 588"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:544"}},"data":"Data for row 589","id":"Row 589"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:545"}},"data":"Data for row 59","id":"Row 59"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:546"}},"data":"Data for row 590","id":"Row 590"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:547"}},"data":"Data for row 591","id":"Row 591"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:548"}},"data":"Data for row 592","id":"Row 592"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:549"}},"data":"Data for row 593","id":"Row 593"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:550"}},"data":"Data for row 594","id":"Row 594"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:551"}},"data":"Data for row 595","id":"Row 595"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:552"}},"data":"Data for row 596","id":"Row 596"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:553"}},"data":"Data for row 597","id":"Row 597"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:554"}},"data":"Data for row 598","id":"Row 598"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:555"}},"data":"Data for row 599","id":"Row 599"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:556"}},"data":"Data for row 6","id":"Row 6"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:557"}},"data":"Data for row 60","id":"Row 60"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:558"}},"data":"Data for row 600","id":"Row 600"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:559"}},"data":"Data for row 601","id":"Row 601"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:560"}},"data":"Data for row 602","id":"Row 602"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:561"}},"data":"Data for row 603","id":"Row 603"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:562"}},"data":"Data for row 604","id":"Row 604"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:563"}},"data":"Data for row 605","id":"Row 605"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:564"}},"data":"Data for row 606","id":"Row 606"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:565"}},"data":"Data for row 607","id":"Row 607"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:566"}},"data":"Data for row 608","id":"Row 608"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:567"}},"data":"Data for row 609","id":"Row 609"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:568"}},"data":"Data for row 61","id":"Row 61"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:569"}},"data":"Data for row 610","id":"Row 610"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:570"}},"data":"Data for row 611","id":"Row 611"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:571"}},"data":"Data for row 612","id":"Row 612"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:572"}},"data":"Data for row 613","id":"Row 613"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:573"}},"data":"Data for row 614","id":"Row 614"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:574"}},"data":"Data for row 615","id":"Row 615"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:575"}},"data":"Data for row 616","id":"Row 616"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:576"}},"data":"Data for row 617","id":"Row 617"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:577"}},"data":"Data for row 618","id":"Row 618"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:578"}},"data":"Data for row 619","id":"Row 619"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:579"}},"data":"Data for row 62","id":"Row 62"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:580"}},"data":"Data for row 620","id":"Row 620"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:581"}},"data":"Data for row 621","id":"Row 621"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:582"}},"data":"Data for row 622","id":"Row 622"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:583"}},"data":"Data for row 623","id":"Row 623"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:584"}},"data":"Data for row 624","id":"Row 624"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:585"}},"data":"Data for row 625","id":"Row 625"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:586"}},"data":"Data for row 626","id":"Row 626"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:587"}},"data":"Data for row 627","id":"Row 627"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:588"}},"data":"Data for row 628","id":"Row 628"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:589"}},"data":"Data for row 629","id":"Row 629"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:590"}},"data":"Data for row 63","id":"Row 63"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:591"}},"data":"Data for row 630","id":"Row 630"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:592"}},"data":"Data for row 631","id":"Row 631"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:593"}},"data":"Data for row 632","id":"Row 632"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:594"}},"data":"Data for row 633","id":"Row 633"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:595"}},"data":"Data for row 634","id":"Row 634"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:596"}},"data":"Data for row 635","id":"Row 635"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:597"}},"data":"Data for row 636","id":"Row 636"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:598"}},"data":"Data for row 637","id":"Row 637"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:599"}},"data":"Data for row 638","id":"Row 638"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:600"}},"data":"Data for row 639","id":"Row 639"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:601"}},"data":"Data for row 64","id":"Row 64"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:602"}},"data":"Data for row 640","id":"Row 640"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:603"}},"data":"Data for row 641","id":"Row 641"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:604"}},"data":"Data for row 642","id":"Row 642"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:605"}},"data":"Data for row 643","id":"Row 643"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:606"}},"data":"Data for row 644","id":"Row 644"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:607"}},"data":"Data for row 645","id":"Row 645"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:608"}},"data":"Data for row 646","id":"Row 646"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:609"}},"data":"Data for row 647","id":"Row 647"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:610"}},"data":"Data for row 648","id":"Row 648"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:611"}},"data":"Data for row 649","id":"Row 649"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:612"}},"data":"Data for row 65","id":"Row 65"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:613"}},"data":"Data for row 650","id":"Row 650"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:614"}},"data":"Data for row 651","id":"Row 651"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:615"}},"data":"Data for row 652","id":"Row 652"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:616"}},"data":"Data for row 653","id":"Row 653"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:617"}},"data":"Data for row 654","id":"Row 654"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:618"}},"data":"Data for row 655","id":"Row 655"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:619"}},"data":"Data for row 656","id":"Row 656"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:620"}},"data":"Data for row 657","id":"Row 657"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:621"}},"data":"Data for row 658","id":"Row 658"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:622"}},"data":"Data for row 659","id":"Row 659"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:623"}},"data":"Data for row 66","id":"Row 66"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:624"}},"data":"Data for row 660","id":"Row 660"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:625"}},"data":"Data for row 661","id":"Row 661"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:626"}},"data":"Data for row 662","id":"Row 662"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:627"}},"data":"Data for row 663","id":"Row 663"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:628"}},"data":"Data for row 664","id":"Row 664"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:629"}},"data":"Data for row 665","id":"Row 665"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:630"}},"data":"Data for row 666","id":"Row 666"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:631"}},"data":"Data for row 667","id":"Row 667"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:632"}},"data":"Data for row 668","id":"Row 668"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:633"}},"data":"Data for row 669","id":"Row 669"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:634"}},"data":"Data for row 67","id":"Row 67"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:635"}},"data":"Data for row 670","id":"Row 670"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:636"}},"data":"Data for row 671","id":"Row 671"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:637"}},"data":"Data for row 672","id":"Row 672"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:638"}},"data":"Data for row 673","id":"Row 673"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:639"}},"data":"Data for row 674","id":"Row 674"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:640"}},"data":"Data for row 675","id":"Row 675"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:641"}},"data":"Data for row 676","id":"Row 676"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:642"}},"data":"Data for row 677","id":"Row 677"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:643"}},"data":"Data for row 678","id":"Row 678"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:644"}},"data":"Data for row 679","id":"Row 679"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:645"}},"data":"Data for row 68","id":"Row 68"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:646"}},"data":"Data for row 680","id":"Row 680"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:647"}},"data":"Data for row 681","id":"Row 681"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:648"}},"data":"Data for row 682","id":"Row 682"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:649"}},"data":"Data for row 683","id":"Row 683"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:650"}},"data":"Data for row 684","id":"Row 684"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:651"}},"data":"Data for row 685","id":"Row 685"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:652"}},"data":"Data for row 686","id":"Row 686"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:653"}},"data":"Data for row 687","id":"Row 687"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:654"}},"data":"Data for row 688","id":"Row 688"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:655"}},"data":"Data for row 689","id":"Row 689"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:656"}},"data":"Data for row 69","id":"Row 69"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:657"}},"data":"Data for row 690","id":"Row 690"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:658"}},"data":"Data for row 691","id":"Row 691"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:659"}},"data":"Data for row 692","id":"Row 692"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:660"}},"data":"Data for row 693","id":"Row 693"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:661"}},"data":"Data for row 694","id":"Row 694"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:662"}},"data":"Data for row 695","id":"Row 695"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:663"}},"data":"Data for row 696","id":"Row 696"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:664"}},"data":"Data for row 697","id":"Row 697"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:665"}},"data":"Data for row 698","id":"Row 698"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:666"}},"data":"Data for row 699","id":"Row 699"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:667"}},"data":"Data for row 7","id":"Row 7"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:668"}},"data":"Data for row 70","id":"Row 70"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:669"}},"data":"Data for row 700","id":"Row 700"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:670"}},"data":"Data for row 701","id":"Row 701"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:671"}},"data":"Data for row 702","id":"Row 702"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:672"}},"data":"Data for row 703","id":"Row 703"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:673"}},"data":"Data for row 704","id":"Row 704"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:674"}},"data":"Data for row 705","id":"Row 705"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:675"}},"data":"Data for row 706","id":"Row 706"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:676"}},"data":"Data for row 707","id":"Row 707"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:677"}},"data":"Data for row 708","id":"Row 708"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:678"}},"data":"Data for row 709","id":"Row 709"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:679"}},"data":"Data for row 71","id":"Row 71"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:680"}},"data":"Data for row 710","id":"Row 710"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:681"}},"data":"Data for row 711","id":"Row 711"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:682"}},"data":"Data for row 712","id":"Row 712"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:683"}},"data":"Data for row 713","id":"Row 713"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:684"}},"data":"Data for row 714","id":"Row 714"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:685"}},"data":"Data for row 715","id":"Row 715"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:686"}},"data":"Data for row 716","id":"Row 716"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:687"}},"data":"Data for row 717","id":"Row 717"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:688"}},"data":"Data for row 718","id":"Row 718"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:689"}},"data":"Data for row 719","id":"Row 719"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:690"}},"data":"Data for row 72","id":"Row 72"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:691"}},"data":"Data for row 720","id":"Row 720"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:692"}},"data":"Data for row 721","id":"Row 721"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:693"}},"data":"Data for row 722","id":"Row 722"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:694"}},"data":"Data for row 723","id":"Row 723"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:695"}},"data":"Data for row 724","id":"Row 724"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:696"}},"data":"Data for row 725","id":"Row 725"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:697"}},"data":"Data for row 726","id":"Row 726"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:698"}},"data":"Data for row 727","id":"Row 727"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:699"}},"data":"Data for row 728","id":"Row 728"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:700"}},"data":"Data for row 729","id":"Row 729"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:701"}},"data":"Data for row 73","id":"Row 73"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:702"}},"data":"Data for row 730","id":"Row 730"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:703"}},"data":"Data for row 731","id":"Row 731"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:704"}},"data":"Data for row 732","id":"Row 732"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:705"}},"data":"Data for row 733","id":"Row 733"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:706"}},"data":"Data for row 734","id":"Row 734"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:707"}},"data":"Data for row 735","id":"Row 735"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:708"}},"data":"Data for row 736","id":"Row 736"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:709"}},"data":"Data for row 737","id":"Row 737"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:710"}},"data":"Data for row 738","id":"Row 738"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:711"}},"data":"Data for row 739","id":"Row 739"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:712"}},"data":"Data for row 74","id":"Row 74"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:713"}},"data":"Data for row 740","id":"Row 740"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:714"}},"data":"Data for row 741","id":"Row 741"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:715"}},"data":"Data for row 742","id":"Row 742"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:716"}},"data":"Data for row 743","id":"Row 743"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:717"}},"data":"Data for row 744","id":"Row 744"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:718"}},"data":"Data for row 745","id":"Row 745"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:719"}},"data":"Data for row 746","id":"Row 746"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:720"}},"data":"Data for row 747","id":"Row 747"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:721"}},"data":"Data for row 748","id":"Row 748"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:722"}},"data":"Data for row 749","id":"Row 749"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:723"}},"data":"Data for row 75","id":"Row 75"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:724"}},"data":"Data for row 750","id":"Row 750"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:725"}},"data":"Data for row 751","id":"Row 751"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:726"}},"data":"Data for row 752","id":"Row 752"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:727"}},"data":"Data for row 753","id":"Row 753"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:728"}},"data":"Data for row 754","id":"Row 754"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:729"}},"data":"Data for row 755","id":"Row 755"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:730"}},"data":"Data for row 756","id":"Row 756"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:731"}},"data":"Data for row 757","id":"Row 757"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:732"}},"data":"Data for row 758","id":"Row 758"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:733"}},"data":"Data for row 759","id":"Row 759"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:734"}},"data":"Data for row 76","id":"Row 76"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:735"}},"data":"Data for row 760","id":"Row 760"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:736"}},"data":"Data for row 761","id":"Row 761"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:737"}},"data":"Data for row 762","id":"Row 762"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:738"}},"data":"Data for row 763","id":"Row 763"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:739"}},"data":"Data for row 764","id":"Row 764"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:740"}},"data":"Data for row 765","id":"Row 765"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:741"}},"data":"Data for row 766","id":"Row 766"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:742"}},"data":"Data for row 767","id":"Row 767"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:743"}},"data":"Data for row 768","id":"Row 768"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:744"}},"data":"Data for row 769","id":"Row 769"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:745"}},"data":"Data for row 77","id":"Row 77"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:746"}},"data":"Data for row 770","id":"Row 770"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:747"}},"data":"Data for row 771","id":"Row 771"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:748"}},"data":"Data for row 772","id":"Row 772"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:749"}},"data":"Data for row 773","id":"Row 773"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:750"}},"data":"Data for row 774","id":"Row 774"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:751"}},"data":"Data for row 775","id":"Row 775"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:752"}},"data":"Data for row 776","id":"Row 776"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:753"}},"data":"Data for row 777","id":"Row 777"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:754"}},"data":"Data for row 778","id":"Row 778"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:755"}},"data":"Data for row 779","id":"Row 779"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:756"}},"data":"Data for row 78","id":"Row 78"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:757"}},"data":"Data for row 780","id":"Row 780"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:758"}},"data":"Data for row 781","id":"Row 781"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:759"}},"data":"Data for row 782","id":"Row 782"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:760"}},"data":"Data for row 783","id":"Row 783"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:761"}},"data":"Data for row 784","id":"Row 784"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:762"}},"data":"Data for row 785","id":"Row 785"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:763"}},"data":"Data for row 786","id":"Row 786"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:764"}},"data":"Data for row 787","id":"Row 787"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:765"}},"data":"Data for row 788","id":"Row 788"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:766"}},"data":"Data for row 789","id":"Row 789"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:767"}},"data":"Data for row 79","id":"Row 79"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:768"}},"data":"Data for row 790","id":"Row 790"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:769"}},"data":"Data for row 791","id":"Row 791"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:770"}},"data":"Data for row 792","id":"Row 792"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:771"}},"data":"Data for row 793","id":"Row 793"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:772"}},"data":"Data for row 794","id":"Row 794"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:773"}},"data":"Data for row 795","id":"Row 795"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:774"}},"data":"Data for row 796","id":"Row 796"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:775"}},"data":"Data for row 797","id":"Row 797"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:776"}},"data":"Data for row 798","id":"Row 798"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:777"}},"data":"Data for row 799","id":"Row 799"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:778"}},"data":"Data for row 8","id":"Row 8"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:779"}},"data":"Data for row 80","id":"Row 80"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:780"}},"data":"Data for row 800","id":"Row 800"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:781"}},"data":"Data for row 801","id":"Row 801"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:782"}},"data":"Data for row 802","id":"Row 802"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:783"}},"data":"Data for row 803","id":"Row 803"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:784"}},"data":"Data for row 804","id":"Row 804"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:785"}},"data":"Data for row 805","id":"Row 805"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:786"}},"data":"Data for row 806","id":"Row 806"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:787"}},"data":"Data for row 807","id":"Row 807"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:788"}},"data":"Data for row 808","id":"Row 808"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:789"}},"data":"Data for row 809","id":"Row 809"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:790"}},"data":"Data for row 81","id":"Row 81"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:791"}},"data":"Data for row 810","id":"Row 810"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:792"}},"data":"Data for row 811","id":"Row 811"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:793"}},"data":"Data for row 812","id":"Row 812"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:794"}},"data":"Data for row 813","id":"Row 813"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:795"}},"data":"Data for row 814","id":"Row 814"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:796"}},"data":"Data for row 815","id":"Row 815"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:797"}},"data":"Data for row 816","id":"Row 816"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:798"}},"data":"Data for row 817","id":"Row 817"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:799"}},"data":"Data for row 818","id":"Row 818"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:800"}},"data":"Data for row 819","id":"Row 819"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:801"}},"data":"Data for row 82","id":"Row 82"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:802"}},"data":"Data for row 820","id":"Row 820"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:803"}},"data":"Data for row 821","id":"Row 821"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:804"}},"data":"Data for row 822","id":"Row 822"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:805"}},"data":"Data for row 823","id":"Row 823"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:806"}},"data":"Data for row 824","id":"Row 824"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:807"}},"data":"Data for row 825","id":"Row 825"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:808"}},"data":"Data for row 826","id":"Row 826"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:809"}},"data":"Data for row 827","id":"Row 827"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:810"}},"data":"Data for row 828","id":"Row 828"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:811"}},"data":"Data for row 829","id":"Row 829"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:812"}},"data":"Data for row 83","id":"Row 83"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:813"}},"data":"Data for row 830","id":"Row 830"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:814"}},"data":"Data for row 831","id":"Row 831"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:815"}},"data":"Data for row 832","id":"Row 832"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:816"}},"data":"Data for row 833","id":"Row 833"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:817"}},"data":"Data for row 834","id":"Row 834"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:818"}},"data":"Data for row 835","id":"Row 835"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:819"}},"data":"Data for row 836","id":"Row 836"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:820"}},"data":"Data for row 837","id":"Row 837"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:821"}},"data":"Data for row 838","id":"Row 838"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:822"}},"data":"Data for row 839","id":"Row 839"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:823"}},"data":"Data for row 84","id":"Row 84"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:824"}},"data":"Data for row 840","id":"Row 840"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:825"}},"data":"Data for row 841","id":"Row 841"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:826"}},"data":"Data for row 842","id":"Row 842"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:827"}},"data":"Data for row 843","id":"Row 843"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:828"}},"data":"Data for row 844","id":"Row 844"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:829"}},"data":"Data for row 845","id":"Row 845"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:830"}},"data":"Data for row 846","id":"Row 846"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:831"}},"data":"Data for row 847","id":"Row 847"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:832"}},"data":"Data for row 848","id":"Row 848"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:833"}},"data":"Data for row 849","id":"Row 849"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:834"}},"data":"Data for row 85","id":"Row 85"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:835"}},"data":"Data for row 850","id":"Row 850"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:836"}},"data":"Data for row 851","id":"Row 851"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:837"}},"data":"Data for row 852","id":"Row 852"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:838"}},"data":"Data for row 853","id":"Row 853"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:839"}},"data":"Data for row 854","id":"Row 854"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:840"}},"data":"Data for row 855","id":"Row 855"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:841"}},"data":"Data for row 856","id":"Row 856"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:842"}},"data":"Data for row 857","id":"Row 857"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:843"}},"data":"Data for row 858","id":"Row 858"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:844"}},"data":"Data for row 859","id":"Row 859"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:845"}},"data":"Data for row 86","id":"Row 86"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:846"}},"data":"Data for row 860","id":"Row 860"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:847"}},"data":"Data for row 861","id":"Row 861"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:848"}},"data":"Data for row 862","id":"Row 862"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:849"}},"data":"Data for row 863","id":"Row 863"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:850"}},"data":"Data for row 864","id":"Row 864"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:851"}},"data":"Data for row 865","id":"Row 865"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:852"}},"data":"Data for row 866","id":"Row 866"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:853"}},"data":"Data for row 867","id":"Row 867"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:854"}},"data":"Data for row 868","id":"Row 868"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:855"}},"data":"Data for row 869","id":"Row 869"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:856"}},"data":"Data for row 87","id":"Row 87"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:857"}},"data":"Data for row 870","id":"Row 870"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:858"}},"data":"Data for row 871","id":"Row 871"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:859"}},"data":"Data for row 872","id":"Row 872"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:860"}},"data":"Data for row 873","id":"Row 873"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:861"}},"data":"Data for row 874","id":"Row 874"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:862"}},"data":"Data for row 875","id":"Row 875"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:863"}},"data":"Data for row 876","id":"Row 876"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:864"}},"data":"Data for row 877","id":"Row 877"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:865"}},"data":"Data for row 878","id":"Row 878"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:866"}},"data":"Data for row 879","id":"Row 879"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:867"}},"data":"Data for row 88","id":"Row 88"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:868"}},"data":"Data for row 880","id":"Row 880"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:869"}},"data":"Data for row 881","id":"Row 881"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:870"}},"data":"Data for row 882","id":"Row 882"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:871"}},"data":"Data for row 883","id":"Row 883"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:872"}},"data":"Data for row 884","id":"Row 884"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:873"}},"data":"Data for row 885","id":"Row 885"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:874"}},"data":"Data for row 886","id":"Row 886"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:875"}},"data":"Data for row 887","id":"Row 887"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:876"}},"data":"Data for row 888","id":"Row 888"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:877"}},"data":"Data for row 889","id":"Row 889"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:878"}},"data":"Data for row 89","id":"Row 89"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:879"}},"data":"Data for row 890","id":"Row 890"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:880"}},"data":"Data for row 891","id":"Row 891"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:881"}},"data":"Data for row 892","id":"Row 892"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:882"}},"data":"Data for row 893","id":"Row 893"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:883"}},"data":"Data for row 894","id":"Row 894"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:884"}},"data":"Data for row 895","id":"Row 895"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:885"}},"data":"Data for row 896","id":"Row 896"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:886"}},"data":"Data for row 897","id":"Row 897"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:887"}},"data":"Data for row 898","id":"Row 898"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:888"}},"data":"Data for row 899","id":"Row 899"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:889"}},"data":"Data for row 9","id":"Row 9"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:890"}},"data":"Data for row 90","id":"Row 90"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:891"}},"data":"Data for row 900","id":"Row 900"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:892"}},"data":"Data for row 901","id":"Row 901"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:893"}},"data":"Data for row 902","id":"Row 902"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:894"}},"data":"Data for row 903","id":"Row 903"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:895"}},"data":"Data for row 904","id":"Row 904"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:896"}},"data":"Data for row 905","id":"Row 905"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:897"}},"data":"Data for row 906","id":"Row 906"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:898"}},"data":"Data for row 907","id":"Row 907"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:899"}},"data":"Data for row 908","id":"Row 908"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:900"}},"data":"Data for row 909","id":"Row 909"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:901"}},"data":"Data for row 91","id":"Row 91"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:902"}},"data":"Data for row 910","id":"Row 910"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:903"}},"data":"Data for row 911","id":"Row 911"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:904"}},"data":"Data for row 912","id":"Row 912"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:905"}},"data":"Data for row 913","id":"Row 913"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:906"}},"data":"Data for row 914","id":"Row 914"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:907"}},"data":"Data for row 915","id":"Row 915"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:908"}},"data":"Data for row 916","id":"Row 916"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:909"}},"data":"Data for row 917","id":"Row 917"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:910"}},"data":"Data for row 918","id":"Row 918"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:911"}},"data":"Data for row 919","id":"Row 919"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:912"}},"data":"Data for row 92","id":"Row 92"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:913"}},"data":"Data for row 920","id":"Row 920"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:914"}},"data":"Data for row 921","id":"Row 921"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:915"}},"data":"Data for row 922","id":"Row 922"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:916"}},"data":"Data for row 923","id":"Row 923"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:917"}},"data":"Data for row 924","id":"Row 924"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:918"}},"data":"Data for row 925","id":"Row 925"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:919"}},"data":"Data for row 926","id":"Row 926"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:920"}},"data":"Data for row 927","id":"Row 927"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:921"}},"data":"Data for row 928","id":"Row 928"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:922"}},"data":"Data for row 929","id":"Row 929"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:923"}},"data":"Data for row 93","id":"Row 93"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:924"}},"data":"Data for row 930","id":"Row 930"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:925"}},"data":"Data for row 931","id":"Row 931"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:926"}},"data":"Data for row 932","id":"Row 932"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:927"}},"data":"Data for row 933","id":"Row 933"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:928"}},"data":"Data for row 934","id":"Row 934"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:929"}},"data":"Data for row 935","id":"Row 935"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:930"}},"data":"Data for row 936","id":"Row 936"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:931"}},"data":"Data for row 937","id":"Row 937"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:932"}},"data":"Data for row 938","id":"Row 938"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:933"}},"data":"Data for row 939","id":"Row 939"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:934"}},"data":"Data for row 94","id":"Row 94"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:935"}},"data":"Data for row 940","id":"Row 940"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:936"}},"data":"Data for row 941","id":"Row 941"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:937"}},"data":"Data for row 942","id":"Row 942"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:938"}},"data":"Data for row 943","id":"Row 943"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:939"}},"data":"Data for row 944","id":"Row 944"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:940"}},"data":"Data for row 945","id":"Row 945"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:941"}},"data":"Data for row 946","id":"Row 946"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:942"}},"data":"Data for row 947","id":"Row 947"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:943"}},"data":"Data for row 948","id":"Row 948"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:944"}},"data":"Data for row 949","id":"Row 949"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:945"}},"data":"Data for row 95","id":"Row 95"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:946"}},"data":"Data for row 950","id":"Row 950"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:947"}},"data":"Data for row 951","id":"Row 951"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:948"}},"data":"Data for row 952","id":"Row 952"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:949"}},"data":"Data for row 953","id":"Row 953"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:950"}},"data":"Data for row 954","id":"Row 954"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:951"}},"data":"Data for row 955","id":"Row 955"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:952"}},"data":"Data for row 956","id":"Row 956"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:953"}},"data":"Data for row 957","id":"Row 957"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:954"}},"data":"Data for row 958","id":"Row 958"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:955"}},"data":"Data for row 959","id":"Row 959"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:956"}},"data":"Data for row 96","id":"Row 96"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:957"}},"data":"Data for row 960","id":"Row 960"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:958"}},"data":"Data for row 961","id":"Row 961"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:959"}},"data":"Data for row 962","id":"Row 962"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:960"}},"data":"Data for row 963","id":"Row 963"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:961"}},"data":"Data for row 964","id":"Row 964"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:962"}},"data":"Data for row 965","id":"Row 965"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:963"}},"data":"Data for row 966","id":"Row 966"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:964"}},"data":"Data for row 967","id":"Row 967"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:965"}},"data":"Data for row 968","id":"Row 968"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:966"}},"data":"Data for row 969","id":"Row 969"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:967"}},"data":"Data for row 97","id":"Row 97"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:968"}},"data":"Data for row 970","id":"Row 970"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:969"}},"data":"Data for row 971","id":"Row 971"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:970"}},"data":"Data for row 972","id":"Row 972"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:971"}},"data":"Data for row 973","id":"Row 973"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:972"}},"data":"Data for row 974","id":"Row 974"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:973"}},"data":"Data for row 975","id":"Row 975"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:974"}},"data":"Data for row 976","id":"Row 976"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:975"}},"data":"Data for row 977","id":"Row 977"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:976"}},"data":"Data for row 978","id":"Row 978"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:977"}},"data":"Data for row 979","id":"Row 979"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:978"}},"data":"Data for row 98","id":"Row 98"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:979"}},"data":"Data for row 980","id":"Row 980"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:980"}},"data":"Data for row 981","id":"Row 981"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:981"}},"data":"Data for row 982","id":"Row 982"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:982"}},"data":"Data for row 983","id":"Row 983"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:983"}},"data":"Data for row 984","id":"Row 984"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:984"}},"data":"Data for row 985","id":"Row 985"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:985"}},"data":"Data for row 986","id":"Row 986"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:986"}},"data":"Data for row 987","id":"Row 987"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:987"}},"data":"Data for row 988","id":"Row 988"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:988"}},"data":"Data for row 989","id":"Row 989"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:989"}},"data":"Data for row 99","id":"Row 99"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:990"}},"data":"Data for row 990","id":"Row 990"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:991"}},"data":"Data for row 991","id":"Row 991"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:992"}},"data":"Data for row 992","id":"Row 992"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:993"}},"data":"Data for row 993","id":"Row 993"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:994"}},"data":"Data for row 994","id":"Row 994"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:995"}},"data":"Data for row 995","id":"Row 995"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:996"}},"data":"Data for row 996","id":"Row 996"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:997"}},"data":"Data for row 997","id":"Row 997"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:998"}},"data":"Data for row 998","id":"Row 998"} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"BackfillModes_25282936","cursor":"backfill:999"}},"data":"Data for row 999","id":"Row 999"} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test%2FBackfillModes_11837744":{"backfilled":1000,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data"],"types":{"data":"text","id":"varchar"}}},"mode":"Active"},"test%2FBackfillModes_25282936":{"backfilled":1000,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data"],"types":{"data":"text","id":"varchar"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} + diff --git a/source-mysql/backfill.go b/source-mysql/backfill.go index c6613e7244..ab64c8ef83 100644 --- a/source-mysql/backfill.go +++ b/source-mysql/backfill.go @@ -50,6 +50,7 @@ func (db *mysqlDatabase) ScanTableChunk(ctx context.Context, info *sqlcapture.Di query = db.keylessScanQuery(info, schema, table) args = []any{state.BackfilledCount} case sqlcapture.TableModePreciseBackfill, sqlcapture.TableModeUnfilteredBackfill: + var isPrecise = (state.Mode == sqlcapture.TableModePreciseBackfill) if resumeAfter != nil { var resumeKey, err = sqlcapture.UnpackTuple(resumeAfter, decodeKeyFDB) if err != nil { @@ -72,13 +73,13 @@ func (db *mysqlDatabase) ScanTableChunk(ctx context.Context, info *sqlcapture.Di for i := range resumeKey { args = append(args, resumeKey[:i+1]...) } - query = db.buildScanQuery(false, keyColumns, columnTypes, schema, table) + query = db.buildScanQuery(false, isPrecise, keyColumns, columnTypes, schema, table) } else { logrus.WithFields(logrus.Fields{ "stream": streamID, "keyColumns": keyColumns, }).Debug("scanning initial table chunk") - query = db.buildScanQuery(true, keyColumns, columnTypes, schema, table) + query = db.buildScanQuery(true, isPrecise, keyColumns, columnTypes, schema, table) } default: return fmt.Errorf("invalid backfill mode %q", state.Mode) @@ -168,12 +169,14 @@ func (db *mysqlDatabase) keylessScanQuery(info *sqlcapture.DiscoveryInfo, schema return query.String() } -func (db *mysqlDatabase) buildScanQuery(start bool, keyColumns []string, columnTypes map[string]interface{}, schemaName, tableName string) string { +func (db *mysqlDatabase) buildScanQuery(start, isPrecise bool, keyColumns []string, columnTypes map[string]interface{}, schemaName, tableName string) string { // Construct lists of key specifiers and placeholders. They will be joined with commas and used in the query itself. var pkey []string for _, colName := range keyColumns { var quotedName = quoteColumnName(colName) - if colType, ok := columnTypes[colName].(string); ok && columnBinaryKeyComparison[colType] { + // If a precise backfill is requested *and* the column type requires binary ordering for precise + // backfill comparisons to work, add the 'BINARY' qualifier to the column name. + if colType, ok := columnTypes[colName].(string); ok && isPrecise && columnBinaryKeyComparison[colType] { pkey = append(pkey, "BINARY "+quotedName) } else { pkey = append(pkey, quotedName) diff --git a/source-mysql/capture_test.go b/source-mysql/capture_test.go index ee3b094a2c..cedd7470a4 100644 --- a/source-mysql/capture_test.go +++ b/source-mysql/capture_test.go @@ -205,3 +205,36 @@ func TestEnumDecodingFix(t *testing.T) { tb.Insert(ctx, t, tableName, [][]interface{}{{11, "A"}, {12, "B"}, {13, "C"}, {14, "D"}, {15, "error"}}) t.Run("replication2", func(t *testing.T) { tests.VerifiedCapture(ctx, t, cs) }) } + +func TestBackfillModes(t *testing.T) { + // Create two tables with 1,000 rows each + var tb, ctx = mysqlTestBackend(t), context.Background() + var uniqueA, uniqueB = "11837744", "25282936" + var tableA = tb.CreateTable(ctx, t, uniqueA, "(id VARCHAR(32) PRIMARY KEY, data TEXT)") + var tableB = tb.CreateTable(ctx, t, uniqueB, "(id VARCHAR(32) PRIMARY KEY, data TEXT)") + + // TODO: Generate more challenging keys? + var rows [][]any + for idx := 0; idx < 1000; idx++ { + rows = append(rows, []any{fmt.Sprintf("Row %d", idx), fmt.Sprintf("Data for row %d", idx)}) + } + tb.Insert(ctx, t, tableA, rows) + tb.Insert(ctx, t, tableB, rows) + + // Capture both tables, one with a precise backfill and one imprecise + var cs = tb.CaptureSpec(ctx, t, regexp.MustCompile(uniqueA), regexp.MustCompile(uniqueB)) + cs.Validator = &st.OrderedCaptureValidator{} + var resA, resB sqlcapture.Resource + require.NoError(t, json.Unmarshal(cs.Bindings[0].ResourceConfigJson, &resA)) + require.NoError(t, json.Unmarshal(cs.Bindings[1].ResourceConfigJson, &resB)) + resA.Mode = sqlcapture.BackfillModeNormal + resB.Mode = sqlcapture.BackfillModePrecise + resourceSpecA, err := json.Marshal(resA) + require.NoError(t, err) + resourceSpecB, err := json.Marshal(resB) + require.NoError(t, err) + cs.Bindings[0].ResourceConfigJson = resourceSpecA + cs.Bindings[1].ResourceConfigJson = resourceSpecB + + tests.VerifiedCapture(ctx, t, cs) +} From f4c7e17434b39804308e1f4f41dd3403785e45b8 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Thu, 14 Mar 2024 16:12:37 +0000 Subject: [PATCH 22/96] sqlcapture: delete reduction for _meta/op: "d" --- sqlcapture/discovery.go | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/sqlcapture/discovery.go b/sqlcapture/discovery.go index 14cb7c6c53..6145744d83 100644 --- a/sqlcapture/discovery.go +++ b/sqlcapture/discovery.go @@ -127,11 +127,40 @@ func DiscoverCatalog(ctx context.Context, db Database) ([]*pc.Response_Discovere Required: []string{"op", "source"}, }, }, - "reduce": map[string]interface{}{ - "strategy": "merge", - }, }, Required: []string{"_meta"}, + If: &jsonschema.Schema{ + Extras: map[string]interface{}{ + "properties": map[string]*jsonschema.Schema{ + "_meta": { + Extras: map[string]interface{}{ + "properties": map[string]*jsonschema.Schema{ + "op": { + Extras: map[string]interface{}{ + "const": "d", + }, + }, + }, + }, + }, + }, + }, + }, + Then: &jsonschema.Schema{ + Extras: map[string]interface{}{ + "reduce": map[string]interface{}{ + "strategy": "merge", + "delete": true, + }, + }, + }, + Else: &jsonschema.Schema{ + Extras: map[string]interface{}{ + "reduce": map[string]interface{}{ + "strategy": "merge", + }, + }, + }, }, {Ref: "#" + anchor}, }, From 8f1dc8a9df9b57d36fc3f1e0c9f0ca88deed9abc Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Thu, 14 Mar 2024 16:23:42 +0000 Subject: [PATCH 23/96] source-mysql: update discovery test snapshots --- .../.snapshots/TestGeneric-KeylessDiscovery | 25 ++++++++-- .../.snapshots/TestGeneric-SimpleDiscovery | 25 ++++++++-- .../.snapshots/TestPartitionedTable-Discovery | 25 ++++++++-- .../TestSecondaryIndexDiscovery-index_only | 25 ++++++++-- ...estSecondaryIndexDiscovery-nonunique_index | 25 ++++++++-- .../TestSecondaryIndexDiscovery-nothing | 25 ++++++++-- ...TestSecondaryIndexDiscovery-nullable_index | 25 ++++++++-- .../TestSecondaryIndexDiscovery-pk_and_index | 25 ++++++++-- .../.snapshots/TestTrickyColumnNames-discover | 50 ++++++++++++++++--- .../.snapshots/TestTrickyTableNames-Discover | 25 ++++++++-- 10 files changed, 242 insertions(+), 33 deletions(-) diff --git a/source-mysql/.snapshots/TestGeneric-KeylessDiscovery b/source-mysql/.snapshots/TestGeneric-KeylessDiscovery index a8d8a486cd..db5872e842 100644 --- a/source-mysql/.snapshots/TestGeneric-KeylessDiscovery +++ b/source-mysql/.snapshots/TestGeneric-KeylessDiscovery @@ -38,6 +38,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -104,9 +126,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-mysql/.snapshots/TestGeneric-SimpleDiscovery b/source-mysql/.snapshots/TestGeneric-SimpleDiscovery index 4c518c63f8..4f1f128bb1 100644 --- a/source-mysql/.snapshots/TestGeneric-SimpleDiscovery +++ b/source-mysql/.snapshots/TestGeneric-SimpleDiscovery @@ -37,6 +37,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -103,9 +125,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-mysql/.snapshots/TestPartitionedTable-Discovery b/source-mysql/.snapshots/TestPartitionedTable-Discovery index a06df21523..5baf8a18bc 100644 --- a/source-mysql/.snapshots/TestPartitionedTable-Discovery +++ b/source-mysql/.snapshots/TestPartitionedTable-Discovery @@ -32,6 +32,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -98,9 +120,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-mysql/.snapshots/TestSecondaryIndexDiscovery-index_only b/source-mysql/.snapshots/TestSecondaryIndexDiscovery-index_only index 89f36af63c..443992d094 100644 --- a/source-mysql/.snapshots/TestSecondaryIndexDiscovery-index_only +++ b/source-mysql/.snapshots/TestSecondaryIndexDiscovery-index_only @@ -38,6 +38,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -104,9 +126,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nonunique_index b/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nonunique_index index 00a25e6240..6e0065f709 100644 --- a/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nonunique_index +++ b/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nonunique_index @@ -35,6 +35,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -101,9 +123,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nothing b/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nothing index 7dee9228e1..6f8c2f3a59 100644 --- a/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nothing +++ b/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nothing @@ -35,6 +35,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -101,9 +123,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nullable_index b/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nullable_index index 7d1c81ae5e..63461bcf97 100644 --- a/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nullable_index +++ b/source-mysql/.snapshots/TestSecondaryIndexDiscovery-nullable_index @@ -41,6 +41,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -107,9 +129,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-mysql/.snapshots/TestSecondaryIndexDiscovery-pk_and_index b/source-mysql/.snapshots/TestSecondaryIndexDiscovery-pk_and_index index 7c97b5a5dd..0c78dfbbde 100644 --- a/source-mysql/.snapshots/TestSecondaryIndexDiscovery-pk_and_index +++ b/source-mysql/.snapshots/TestSecondaryIndexDiscovery-pk_and_index @@ -34,6 +34,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -100,9 +122,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-mysql/.snapshots/TestTrickyColumnNames-discover b/source-mysql/.snapshots/TestTrickyColumnNames-discover index e41114cccd..f88d0fd8ec 100644 --- a/source-mysql/.snapshots/TestTrickyColumnNames-discover +++ b/source-mysql/.snapshots/TestTrickyColumnNames-discover @@ -28,6 +28,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -94,9 +116,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { @@ -138,6 +157,28 @@ Binding 1: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -204,9 +245,6 @@ Binding 1: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-mysql/.snapshots/TestTrickyTableNames-Discover b/source-mysql/.snapshots/TestTrickyTableNames-Discover index 7dded51a9f..f57f2f1f33 100644 --- a/source-mysql/.snapshots/TestTrickyTableNames-Discover +++ b/source-mysql/.snapshots/TestTrickyTableNames-Discover @@ -25,6 +25,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -91,9 +113,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { From 0896361ac7c364919b0d782a95da5ce831047be0 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Thu, 14 Mar 2024 16:42:47 +0000 Subject: [PATCH 24/96] source-sqlserver: update discover test snapshots --- .../.snapshots/TestIndexIncludedDiscovery | 25 ++++++++++++++++--- .../TestSecondaryIndexDiscovery-index_only | 25 ++++++++++++++++--- ...estSecondaryIndexDiscovery-nonunique_index | 25 ++++++++++++++++--- .../TestSecondaryIndexDiscovery-nothing | 25 ++++++++++++++++--- ...TestSecondaryIndexDiscovery-nullable_index | 25 ++++++++++++++++--- .../TestSecondaryIndexDiscovery-pk_and_index | 25 ++++++++++++++++--- .../.snapshots/TestVarcharKeyDiscovery | 25 ++++++++++++++++--- 7 files changed, 154 insertions(+), 21 deletions(-) diff --git a/source-sqlserver/.snapshots/TestIndexIncludedDiscovery b/source-sqlserver/.snapshots/TestIndexIncludedDiscovery index 9eab9b2c2a..620911a72b 100644 --- a/source-sqlserver/.snapshots/TestIndexIncludedDiscovery +++ b/source-sqlserver/.snapshots/TestIndexIncludedDiscovery @@ -38,6 +38,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -110,9 +132,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-index_only b/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-index_only index 301fd4036f..4ec0a281f2 100644 --- a/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-index_only +++ b/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-index_only @@ -38,6 +38,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -110,9 +132,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nonunique_index b/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nonunique_index index 86afcaced7..e2588ba8e8 100644 --- a/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nonunique_index +++ b/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nonunique_index @@ -35,6 +35,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -107,9 +129,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nothing b/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nothing index 52a2f642ac..a08e05ab90 100644 --- a/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nothing +++ b/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nothing @@ -35,6 +35,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -107,9 +129,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nullable_index b/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nullable_index index dc92bd31a7..e11a824661 100644 --- a/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nullable_index +++ b/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-nullable_index @@ -41,6 +41,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -113,9 +135,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-pk_and_index b/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-pk_and_index index 67e676d944..4572323401 100644 --- a/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-pk_and_index +++ b/source-sqlserver/.snapshots/TestSecondaryIndexDiscovery-pk_and_index @@ -34,6 +34,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -106,9 +128,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-sqlserver/.snapshots/TestVarcharKeyDiscovery b/source-sqlserver/.snapshots/TestVarcharKeyDiscovery index b98fa33943..e7a69d1706 100644 --- a/source-sqlserver/.snapshots/TestVarcharKeyDiscovery +++ b/source-sqlserver/.snapshots/TestVarcharKeyDiscovery @@ -28,6 +28,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -100,9 +122,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { From 9c349278134175305419bb8542a591f88352bab0 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Thu, 14 Mar 2024 12:22:23 +0000 Subject: [PATCH 25/96] ci: only refresh tags if connector code is updated --- .github/actions/deploy/action.yaml | 9 ++++++++- .github/workflows/ci.yaml | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/actions/deploy/action.yaml b/.github/actions/deploy/action.yaml index 6d6d6d0fc3..c6eaa760ce 100644 --- a/.github/actions/deploy/action.yaml +++ b/.github/actions/deploy/action.yaml @@ -60,8 +60,15 @@ runs: shell: bash run: sudo apt install postgresql + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + connector: + - "${{ inputs.connector }}/**" + - name: Refresh connector tags for ${{ inputs.connector }} - if: ${{ github.event_name == 'push' }} + if: github.event_name == 'push' && steps.filter.outputs.connector == 'true' shell: bash env: PGDATABASE: ${{ inputs.pg_database }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9745cb4add..5fc14444eb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -301,8 +301,15 @@ jobs: sudo apt update sudo apt install postgresql + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + connector: + - "${{ matrix.connector }}/**" + - name: Refresh connector tags for ${{ matrix.connector }} - if: ${{ github.event_name == 'push' }} + if: github.event_name == 'push' && steps.filter.outputs.connector == 'true' env: PGHOST: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }} PGUSER: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }} From f395ab46d53efbe0c6438f87d38c0acd90e5b797 Mon Sep 17 00:00:00 2001 From: Joseph Shearer Date: Mon, 18 Mar 2024 20:46:19 -0400 Subject: [PATCH 26/96] Set a default for `next_page` --- estuary-cdk/estuary_cdk/capture/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/estuary-cdk/estuary_cdk/capture/common.py b/estuary-cdk/estuary_cdk/capture/common.py index 6d8e7da94b..8d789716f2 100644 --- a/estuary-cdk/estuary_cdk/capture/common.py +++ b/estuary-cdk/estuary_cdk/capture/common.py @@ -133,7 +133,8 @@ class Backfill(BaseModel, extra="forbid"): description="LogCursor at which incremental replication began" ) next_page: PageCursor = Field( - description="PageCursor of the next page to fetch" + description="PageCursor of the next page to fetch", + default=None ) class Snapshot(BaseModel, extra="forbid"): From 0bba1280f32955dba700c211e42b5b8c94bf77a5 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Tue, 19 Mar 2024 17:06:52 -0500 Subject: [PATCH 27/96] source-boilerplate: Buffer documents to amortize write overhead This change is fairly small but impactful. Johnny's profiling earlier today apparently found that we're spending a lot of time blocked on write syscalls, which makes sense since we currently output every document to Flow as a separate message send. This commit wraps a 1MiB `bufio.Writer` around stdout at the very base level, and adds flush logic to the `Send()` method so that anything *except* a document flushes that buffer. The end result should be that our behavior is mostly unchanged except that we'll accumulate capture documents into batches which get flushed only when the buffer fills up or we reach the next checkpoint. This should hopefully improve connector throughput, though I can't say by how much. This is a very low-level change, so it isn't exercised by most of our CI test suite. However our integration tests should run the new logic, so if CI passes we've probably got it right. Also Johnny wrote a very similar implementation in parallel before he had to leave for his vacation (see https://github.com/estuary/connectors/pull/1388) so comparing the two also gives me some confidence this is correct. --- source-boilerplate/boilerplate.go | 40 ++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/source-boilerplate/boilerplate.go b/source-boilerplate/boilerplate.go index 39bf6cca2d..e5abd87f36 100644 --- a/source-boilerplate/boilerplate.go +++ b/source-boilerplate/boilerplate.go @@ -29,6 +29,8 @@ import ( "google.golang.org/grpc/metadata" ) +const outputWriteBuffer = 1 * 1024 * 1024 // 1MiB output buffer to amortize write syscall overhead + // StateKey is used to key binding-specific state a connector's driver checkpoint. It's just a type // alias for a string to help differentiate any old string from where a specific state key from the // runtime should be used. @@ -273,17 +275,20 @@ func (out *PullOutput) DocumentsAndCheckpoint(checkpoint json.RawMessage, merge } func newProtoCodec(ctx context.Context) pc.Connector_CaptureServer { + var bw = bufio.NewWriterSize(os.Stdout, outputWriteBuffer) return &protoCodec{ ctx: ctx, r: bufio.NewReaderSize(os.Stdin, 1<<21), - w: protoio.NewUint32DelimitedWriter(os.Stdout, binary.LittleEndian), + bw: bw, + pw: protoio.NewUint32DelimitedWriter(bw, binary.LittleEndian), } } type protoCodec struct { ctx context.Context r *bufio.Reader - w protoio.Writer + bw *bufio.Writer + pw protoio.Writer } func (c *protoCodec) Context() context.Context { @@ -291,7 +296,17 @@ func (c *protoCodec) Context() context.Context { } func (c *protoCodec) Send(m *pc.Response) error { - return c.SendMsg(m) + if err := c.SendMsg(m); err != nil { + return err + } + + // Flush the buffered output writer after anything *other* than a captured document. + // This logic assumes that either a captured document is present or this is some + // other sort of message, which is always true in practice. + if m.Captured == nil { + return c.bw.Flush() + } + return nil } func (c *protoCodec) Recv() (*pc.Request, error) { var m = new(pc.Request) @@ -301,7 +316,7 @@ func (c *protoCodec) Recv() (*pc.Request, error) { return m, nil } func (c *protoCodec) SendMsg(m interface{}) error { - return c.w.WriteMsg(m.(proto.Message)) + return c.pw.WriteMsg(m.(proto.Message)) } func (c *protoCodec) RecvMsg(m interface{}) error { var lengthBytes [4]byte @@ -373,6 +388,7 @@ func (c *protoCodec) peekMessage(size int) ([]byte, error) { } func newJsonCodec(ctx context.Context) pc.Connector_CaptureServer { + var bw = bufio.NewWriterSize(os.Stdout, outputWriteBuffer) return &jsonCodec{ ctx: ctx, marshaler: jsonpb.Marshaler{ @@ -387,6 +403,7 @@ func newJsonCodec(ctx context.Context) pc.Connector_CaptureServer { AnyResolver: nil, }, decoder: json.NewDecoder(bufio.NewReaderSize(os.Stdin, 1<<21)), + bw: bw, } } @@ -395,13 +412,24 @@ type jsonCodec struct { marshaler jsonpb.Marshaler unmarshaler jsonpb.Unmarshaler decoder *json.Decoder + bw *bufio.Writer } func (c *jsonCodec) Context() context.Context { return c.ctx } func (c *jsonCodec) Send(m *pc.Response) error { - return c.SendMsg(m) + if err := c.SendMsg(m); err != nil { + return err + } + + // Flush the buffered output writer after anything *other* than a captured document. + // This logic assumes that either a captured document is present or this is some + // other sort of message, which is always true in practice. + if m.Captured == nil { + return c.bw.Flush() + } + return nil } func (c *jsonCodec) Recv() (*pc.Request, error) { var m = new(pc.Request) @@ -418,7 +446,7 @@ func (c *jsonCodec) SendMsg(m interface{}) error { } _ = w.WriteByte('\n') - var _, err = os.Stdout.Write(w.Bytes()) + var _, err = c.bw.Write(w.Bytes()) return err } func (c *jsonCodec) RecvMsg(m interface{}) error { From 59cefb713301ab4b37262be6153472ea8dbc139c Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Tue, 19 Mar 2024 15:38:05 +0000 Subject: [PATCH 28/96] facebook: use default empty string for primary keys instead of required --- .../streams/base_insight_streams.py | 13 +- .../snapshots__discover__capture.stdout.json | 129 +++++++++++------- 2 files changed, 84 insertions(+), 58 deletions(-) diff --git a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 6c248a85e6..a55e23c8d6 100644 --- a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -301,16 +301,9 @@ def get_json_schema(self) -> Mapping[str, Any]: if k not in schema["properties"]: continue - tys = schema["properties"][k]["type"] - - if "null" in tys: - schema["properties"][k]["type"].remove("null") - - if "required" not in schema: - schema["required"] = [] - - if k not in schema["required"]: - schema["required"].append(k) + typ = schema["properties"][k]["type"] + if "string" in typ or typ == "string": + schema["properties"][k]["default"] = "" return schema diff --git a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json index 050e256c63..cc3b7fb179 100644 --- a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json @@ -12218,7 +12218,8 @@ "account_id": { "type": [ "string" - ] + ], + "default": "" }, "account_name": { "type": [ @@ -12451,7 +12452,8 @@ "ad_id": { "type": [ "string" - ] + ], + "default": "" }, "ad_impression_actions": { "type": [ @@ -14014,7 +14016,8 @@ "format": "date", "type": [ "string" - ] + ], + "default": "" }, "date_stop": { "format": "date", @@ -15951,7 +15954,8 @@ "account_id": { "type": [ "string" - ] + ], + "default": "" }, "account_name": { "type": [ @@ -16184,7 +16188,8 @@ "ad_id": { "type": [ "string" - ] + ], + "default": "" }, "ad_impression_actions": { "type": [ @@ -17747,7 +17752,8 @@ "format": "date", "type": [ "string" - ] + ], + "default": "" }, "date_stop": { "format": "date", @@ -19639,13 +19645,17 @@ }, "age": { "type": [ + "null", "string" - ] + ], + "default": "" }, "gender": { "type": [ + "null", "string" - ] + ], + "default": "" }, "_meta": { "type": "object", @@ -19665,9 +19675,7 @@ "required": [ "account_id", "ad_id", - "date_start", - "age", - "gender" + "date_start" ] }, "key": [ @@ -19698,7 +19706,8 @@ "account_id": { "type": [ "string" - ] + ], + "default": "" }, "account_name": { "type": [ @@ -19931,7 +19940,8 @@ "ad_id": { "type": [ "string" - ] + ], + "default": "" }, "ad_impression_actions": { "type": [ @@ -21494,7 +21504,8 @@ "format": "date", "type": [ "string" - ] + ], + "default": "" }, "date_stop": { "format": "date", @@ -23386,8 +23397,10 @@ }, "country": { "type": [ + "null", "string" - ] + ], + "default": "" }, "_meta": { "type": "object", @@ -23407,8 +23420,7 @@ "required": [ "account_id", "ad_id", - "date_start", - "country" + "date_start" ] }, "key": [ @@ -23438,7 +23450,8 @@ "account_id": { "type": [ "string" - ] + ], + "default": "" }, "account_name": { "type": [ @@ -23671,7 +23684,8 @@ "ad_id": { "type": [ "string" - ] + ], + "default": "" }, "ad_impression_actions": { "type": [ @@ -25234,7 +25248,8 @@ "format": "date", "type": [ "string" - ] + ], + "default": "" }, "date_stop": { "format": "date", @@ -27126,8 +27141,10 @@ }, "region": { "type": [ + "null", "string" - ] + ], + "default": "" }, "_meta": { "type": "object", @@ -27147,8 +27164,7 @@ "required": [ "account_id", "ad_id", - "date_start", - "region" + "date_start" ] }, "key": [ @@ -27178,7 +27194,8 @@ "account_id": { "type": [ "string" - ] + ], + "default": "" }, "account_name": { "type": [ @@ -27411,7 +27428,8 @@ "ad_id": { "type": [ "string" - ] + ], + "default": "" }, "ad_impression_actions": { "type": [ @@ -28974,7 +28992,8 @@ "format": "date", "type": [ "string" - ] + ], + "default": "" }, "date_stop": { "format": "date", @@ -30866,8 +30885,10 @@ }, "dma": { "type": [ + "null", "string" - ] + ], + "default": "" }, "_meta": { "type": "object", @@ -30887,8 +30908,7 @@ "required": [ "account_id", "ad_id", - "date_start", - "dma" + "date_start" ] }, "key": [ @@ -30918,7 +30938,8 @@ "account_id": { "type": [ "string" - ] + ], + "default": "" }, "account_name": { "type": [ @@ -31151,7 +31172,8 @@ "ad_id": { "type": [ "string" - ] + ], + "default": "" }, "ad_impression_actions": { "type": [ @@ -32714,7 +32736,8 @@ "format": "date", "type": [ "string" - ] + ], + "default": "" }, "date_stop": { "format": "date", @@ -34606,18 +34629,24 @@ }, "publisher_platform": { "type": [ + "null", "string" - ] + ], + "default": "" }, "platform_position": { "type": [ + "null", "string" - ] + ], + "default": "" }, "impression_device": { "type": [ + "null", "string" - ] + ], + "default": "" }, "_meta": { "type": "object", @@ -34637,10 +34666,7 @@ "required": [ "account_id", "ad_id", - "date_start", - "publisher_platform", - "platform_position", - "impression_device" + "date_start" ] }, "key": [ @@ -34672,7 +34698,8 @@ "account_id": { "type": [ "string" - ] + ], + "default": "" }, "account_name": { "type": [ @@ -34905,7 +34932,8 @@ "ad_id": { "type": [ "string" - ] + ], + "default": "" }, "ad_impression_actions": { "type": [ @@ -36468,7 +36496,8 @@ "format": "date", "type": [ "string" - ] + ], + "default": "" }, "date_stop": { "format": "date", @@ -39183,7 +39212,8 @@ "account_id": { "type": [ "string" - ] + ], + "default": "" }, "account_name": { "type": [ @@ -39194,7 +39224,8 @@ "ad_id": { "type": [ "string" - ] + ], + "default": "" }, "ad_name": { "type": [ @@ -39236,7 +39267,8 @@ "format": "date", "type": [ "string" - ] + ], + "default": "" }, "date_stop": { "format": "date", @@ -39339,8 +39371,10 @@ }, "publisher_platform": { "type": [ + "null", "string" - ] + ], + "default": "" }, "_meta": { "type": "object", @@ -39360,8 +39394,7 @@ "required": [ "account_id", "ad_id", - "date_start", - "publisher_platform" + "date_start" ] }, "key": [ From a3a0e4be98f7bb0f211733eb4af7e5e78105c3ab Mon Sep 17 00:00:00 2001 From: Jon Wihl Date: Tue, 19 Mar 2024 21:05:21 -0400 Subject: [PATCH 29/96] Prevent string default value for date-time fields --- .../streams/base_insight_streams.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index a55e23c8d6..dfc111ccbd 100644 --- a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -300,10 +300,11 @@ def get_json_schema(self) -> Mapping[str, Any]: for k in self.primary_key: if k not in schema["properties"]: continue - + typ = schema["properties"][k]["type"] if "string" in typ or typ == "string": - schema["properties"][k]["default"] = "" + if "format" not in schema["properties"][k]: + schema["properties"][k]["default"] = "" return schema From d7d6473051ac824a5c773ad6b8620f783b04100e Mon Sep 17 00:00:00 2001 From: Jon Wihl Date: Tue, 19 Mar 2024 23:04:20 -0400 Subject: [PATCH 30/96] updated snapshot testing --- source-facebook-marketing/test.flow.yaml | 34 +++++++++++++++++-- .../snapshots__discover__capture.stdout.json | 24 +++++-------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/source-facebook-marketing/test.flow.yaml b/source-facebook-marketing/test.flow.yaml index 6ca9daf1db..c599739449 100644 --- a/source-facebook-marketing/test.flow.yaml +++ b/source-facebook-marketing/test.flow.yaml @@ -4,8 +4,6 @@ import: captures: acmeCo/source-facebook-marketing: endpoint: - # connector: - # image: ghcr.io/estuary/source-facebook-marketing:testing local: command: - python @@ -20,10 +18,14 @@ captures: - resource: stream: ad_sets syncMode: incremental + cursorField: + - updated_time target: acmeCo/ad_sets - resource: stream: ads syncMode: incremental + cursorField: + - updated_time target: acmeCo/ads - resource: stream: ad_creatives @@ -32,34 +34,50 @@ captures: - resource: stream: ads_insights syncMode: incremental + cursorField: + - date_start target: acmeCo/ads_insights - resource: stream: ads_insights_age_and_gender syncMode: incremental + cursorField: + - date_start target: acmeCo/ads_insights_age_and_gender - resource: stream: ads_insights_country syncMode: incremental + cursorField: + - date_start target: acmeCo/ads_insights_country - resource: stream: ads_insights_region syncMode: incremental + cursorField: + - date_start target: acmeCo/ads_insights_region - resource: stream: ads_insights_dma syncMode: incremental + cursorField: + - date_start target: acmeCo/ads_insights_dma - resource: stream: ads_insights_platform_and_device syncMode: incremental + cursorField: + - date_start target: acmeCo/ads_insights_platform_and_device - resource: stream: ads_insights_action_type syncMode: incremental + cursorField: + - date_start target: acmeCo/ads_insights_action_type - resource: stream: campaigns syncMode: incremental + cursorField: + - updated_time target: acmeCo/campaigns - resource: stream: custom_conversions @@ -68,12 +86,24 @@ captures: - resource: stream: images syncMode: incremental + cursorField: + - updated_time target: acmeCo/images - resource: stream: videos syncMode: incremental + cursorField: + - updated_time target: acmeCo/videos + - resource: + stream: activities + syncMode: incremental + cursorField: + - event_time + target: acmeCo/activities - resource: stream: customads_insights_publisher_platform syncMode: incremental + cursorField: + - date_start target: acmeCo/customads_insights_publisher_platform diff --git a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json index cc3b7fb179..44486b7a68 100644 --- a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json @@ -14016,8 +14016,7 @@ "format": "date", "type": [ "string" - ], - "default": "" + ] }, "date_stop": { "format": "date", @@ -17752,8 +17751,7 @@ "format": "date", "type": [ "string" - ], - "default": "" + ] }, "date_stop": { "format": "date", @@ -21504,8 +21502,7 @@ "format": "date", "type": [ "string" - ], - "default": "" + ] }, "date_stop": { "format": "date", @@ -25248,8 +25245,7 @@ "format": "date", "type": [ "string" - ], - "default": "" + ] }, "date_stop": { "format": "date", @@ -28992,8 +28988,7 @@ "format": "date", "type": [ "string" - ], - "default": "" + ] }, "date_stop": { "format": "date", @@ -32736,8 +32731,7 @@ "format": "date", "type": [ "string" - ], - "default": "" + ] }, "date_stop": { "format": "date", @@ -36496,8 +36490,7 @@ "format": "date", "type": [ "string" - ], - "default": "" + ] }, "date_stop": { "format": "date", @@ -39267,8 +39260,7 @@ "format": "date", "type": [ "string" - ], - "default": "" + ] }, "date_stop": { "format": "date", From cf54d1b94ff5c8e2806a7f6fb94f29b55b8d742f Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 20 Mar 2024 14:30:36 +0000 Subject: [PATCH 31/96] facebook: use string formatting for errors pydantic cannot serialize job as JSON, and ends up throwing the error: TypeError: Object of type 'InsightAsyncJob' is not JSON serializable --- .../source_facebook_marketing/streams/async_job_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py b/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py index 8bfcc6fe74..b0e460c196 100644 --- a/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py +++ b/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py @@ -103,13 +103,13 @@ def _check_jobs_status_and_restart(self) -> List[AsyncJob]: if job.attempt_number >= self.MAX_NUMBER_OF_ATTEMPTS: raise JobException(f"{job}: failed more than {self.MAX_NUMBER_OF_ATTEMPTS} times. Terminating...") elif job.attempt_number == 2: - logger.info("%s: failed second time, trying to split job into smaller jobs.", job) + logger.info(f"{job}: failed second time, trying to split job into smaller jobs.") smaller_jobs = job.split_job() grouped_jobs = ParentAsyncJob(api=self._api.api, jobs=smaller_jobs, interval=job.interval) running_jobs.append(grouped_jobs) grouped_jobs.start() else: - logger.info("%s: failed, restarting", job) + logger.info(f"{job}: failed, restarting") job.restart() running_jobs.append(job) failed_num += 1 From 7c9f1967ac9a7b5cd0f63c01254b36f09be76bd4 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 20 Mar 2024 15:42:16 +0000 Subject: [PATCH 32/96] tests/databricks: fetch rows using code to represent nulls properly --- .../materialize-databricks/fetch-data.go | 107 ++ .../materialize-databricks/fetch.sh | 9 +- .../materialize-databricks/snapshot.json | 1027 ++++++++--------- 3 files changed, 562 insertions(+), 581 deletions(-) create mode 100644 tests/materialize/materialize-databricks/fetch-data.go diff --git a/tests/materialize/materialize-databricks/fetch-data.go b/tests/materialize/materialize-databricks/fetch-data.go new file mode 100644 index 0000000000..d60c3294a3 --- /dev/null +++ b/tests/materialize/materialize-databricks/fetch-data.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + "log" + "math" + "net/url" + "os" + "strings" + + _ "github.com/databricks/databricks-sql-go" +) + +const defaultPort = "443" + +func mustDSN() string { + var accessToken = os.Getenv("DATABRICKS_ACCESS_TOKEN") + var catalog = os.Getenv("DATABRICKS_CATALOG") + var address = os.Getenv("DATABRICKS_HOST_NAME") + if !strings.Contains(address, ":") { + address = address + ":" + defaultPort + } + var schema = os.Getenv("DATABRICKS_SCHEMA") + var httpPath = os.Getenv("DATABRICKS_HTTP_PATH") + + var params = make(url.Values) + params.Add("catalog", catalog) + params.Add("schema", schema) + params.Add("userAgentEntry", "Estuary Technologies Flow") + + var uri = url.URL{ + Host: address, + Path: httpPath, + User: url.UserPassword("token", accessToken), + RawQuery: params.Encode(), + } + + return strings.TrimLeft(uri.String(), "/") +} + +func main() { + flag.Parse() + + tables := flag.Args() + if len(tables) != 1 { + log.Fatal("must provide single table name as an argument") + } + + ctx := context.Background() + + db, err := sql.Open("databricks", mustDSN()) + if err != nil { + log.Fatal(fmt.Errorf("sql.Open: %w", err)) + } + defer db.Close() + + query := fmt.Sprintf("SELECT * FROM %s ORDER BY id, flow_published_at;", tables[0]) + + rows, err := db.QueryContext(ctx, query) + if err != nil { + log.Fatal(fmt.Errorf("queryContext %q: %w", query, err)) + } + + cols, err := rows.Columns() + if err != nil { + log.Fatal(fmt.Errorf("getting columns: %w", err)) + } + + data := make([]interface{}, len(cols)) + ptrs := make([]interface{}, len(cols)) + for i := range data { + ptrs[i] = &data[i] + } + + queriedRows := []map[string]any{} + + for rows.Next() { + if err = rows.Scan(ptrs...); err != nil { + log.Fatal("scanning row: %w", err) + } + row := make(map[string]any) + for idx, val := range data { + switch v := val.(type) { + case float64: + if math.IsNaN(v) { + val = "NaN" + } else if math.IsInf(v, +1) { + val = "Infinity" + } else if math.IsInf(v, -1) { + val = "-Infinity" + } + } + row[cols[idx]] = val + } + + queriedRows = append(queriedRows, row) + } + rows.Close() + + if err := json.NewEncoder(os.Stdout).Encode(queriedRows); err != nil { + log.Fatal(fmt.Errorf("writing output: %w", err)) + } +} diff --git a/tests/materialize/materialize-databricks/fetch.sh b/tests/materialize/materialize-databricks/fetch.sh index 199b9a13b8..178c3bd714 100755 --- a/tests/materialize/materialize-databricks/fetch.sh +++ b/tests/materialize/materialize-databricks/fetch.sh @@ -5,7 +5,7 @@ set -o pipefail set -o nounset function exportToJsonl() { - dbsqlcli -e "SELECT * FROM $1;" | mlr --icsv --ojsonl cat | jq -c "{ table: \"$1\", row: . }" + go run ${TEST_DIR}/materialize-databricks/fetch-data.go "$1" | jq "{ \"table\": \"$1\", rows: . }" } exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.simple" @@ -13,9 +13,4 @@ exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.duplicate_keys_standard" exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.duplicate_keys_delta" exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.duplicate_keys_delta_exclude_flow_doc" exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.multiple_types" - -# due to a bug in the cli, we can't use this at the moment, -# see https://github.com/databricks/databricks-sql-cli/issues/51 -# exportToJsonl "\`$DATABRICKS_CATALOG\`.default.formatted_strings" -dbsqlcli -e "SELECT id, datetime::string, date::string, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document FROM main.\`some-schema\`.formatted_strings;" | \ - mlr --icsv --ojsonl cat | jq -c "del(.flow_document) | { table: \"\`$DATABRICKS_CATALOG\`.\`some-schema\`.formatted_strings\", row: . }" +exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.formatted_strings" diff --git a/tests/materialize/materialize-databricks/snapshot.json b/tests/materialize/materialize-databricks/snapshot.json index 5d9f28500b..0e05ddc37e 100644 --- a/tests/materialize/materialize-databricks/snapshot.json +++ b/tests/materialize/materialize-databricks/snapshot.json @@ -330,595 +330,474 @@ } } { - "row": { - "canary": "amputation's", - "flow_document": "{\"_meta\":{\"uuid\":\"75c06bd6-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"amputation's\",\"id\":1}", - "flow_published_at": "2023-07-12 18:18:11.537199+00:00", - "id": 1 - }, - "table": "`main`.`some-schema`.simple" -} -{ - "row": { - "canary": "armament's", - "flow_document": "{\"_meta\":{\"uuid\":\"7dbe8ebc-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"armament's\",\"id\":2}", - "flow_published_at": "2023-07-12 18:18:24.946758+00:00", - "id": 2 - }, - "table": "`main`.`some-schema`.simple" -} -{ - "row": { - "canary": "splatters", - "flow_document": "{\"_meta\":{\"uuid\":\"8bbf898a-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"splatters\",\"id\":3}", - "flow_published_at": "2023-07-12 18:18:48.441281+00:00", - "id": 3 - }, - "table": "`main`.`some-schema`.simple" -} -{ - "row": { - "canary": "strengthen", - "flow_document": "{\"_meta\":{\"uuid\":\"9b994ae4-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"strengthen\",\"id\":4}", - "flow_published_at": "2023-07-12 18:19:15.034186+00:00", - "id": 4 - }, - "table": "`main`.`some-schema`.simple" -} -{ - "row": { - "canary": "Kringle's", - "flow_document": "{\"_meta\":{\"uuid\":\"b4b60968-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"Kringle's\",\"id\":5}", - "flow_published_at": "2023-07-12 18:19:57.165604+00:00", - "id": 5 - }, - "table": "`main`.`some-schema`.simple" -} -{ - "row": { - "canary": "grosbeak's", - "flow_document": "{\"_meta\":{\"uuid\":\"bcef4bc6-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"grosbeak's\",\"id\":6}", - "flow_published_at": "2023-07-12 18:20:10.962631+00:00", - "id": 6 - }, - "table": "`main`.`some-schema`.simple" -} -{ - "row": { - "canary": "pieced", - "flow_document": "{\"_meta\":{\"uuid\":\"134a1456-20e1-11ee-990b-ffd12dfcd47f\"},\"canary\":\"pieced\",\"id\":7}", - "flow_published_at": "2023-07-12 18:22:35.841647+00:00", - "id": 7 - }, - "table": "`main`.`some-schema`.simple" -} -{ - "row": { - "canary": "roaches", - "flow_document": "{\"_meta\":{\"uuid\":\"1b953992-20e1-11ee-990b-ffd12dfcd47f\"},\"canary\":\"roaches\",\"id\":8}", - "flow_published_at": "2023-07-12 18:22:49.755893+00:00", - "id": 8 - }, - "table": "`main`.`some-schema`.simple" -} -{ - "row": { - "canary": "devilish", - "flow_document": "{\"_meta\":{\"uuid\":\"7e2df422-20e1-11ee-990b-ffd12dfcd47f\"},\"canary\":\"devilish\",\"id\":9}", - "flow_published_at": "2023-07-12 18:25:35.173533+00:00", - "id": 9 - }, - "table": "`main`.`some-schema`.simple" -} -{ - "row": { - "canary": "glucose's", - "flow_document": "{\"_meta\":{\"uuid\":\"8458b580-20e1-11ee-990b-ffd12dfcd47f\"},\"canary\":\"glucose's\",\"id\":10}", - "flow_published_at": "2023-07-12 18:25:45.520064+00:00", - "id": 10 - }, + "rows": [ + { + "canary": "amputation's", + "flow_document": "{\"_meta\":{\"uuid\":\"75c06bd6-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"amputation's\",\"id\":1}", + "flow_published_at": "2023-07-12T18:18:11.537199Z", + "id": 1 + }, + { + "canary": "armament's", + "flow_document": "{\"_meta\":{\"uuid\":\"7dbe8ebc-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"armament's\",\"id\":2}", + "flow_published_at": "2023-07-12T18:18:24.946758Z", + "id": 2 + }, + { + "canary": "splatters", + "flow_document": "{\"_meta\":{\"uuid\":\"8bbf898a-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"splatters\",\"id\":3}", + "flow_published_at": "2023-07-12T18:18:48.441281Z", + "id": 3 + }, + { + "canary": "strengthen", + "flow_document": "{\"_meta\":{\"uuid\":\"9b994ae4-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"strengthen\",\"id\":4}", + "flow_published_at": "2023-07-12T18:19:15.034186Z", + "id": 4 + }, + { + "canary": "Kringle's", + "flow_document": "{\"_meta\":{\"uuid\":\"b4b60968-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"Kringle's\",\"id\":5}", + "flow_published_at": "2023-07-12T18:19:57.165604Z", + "id": 5 + }, + { + "canary": "grosbeak's", + "flow_document": "{\"_meta\":{\"uuid\":\"bcef4bc6-20e0-11ee-990b-ffd12dfcd47f\"},\"canary\":\"grosbeak's\",\"id\":6}", + "flow_published_at": "2023-07-12T18:20:10.962631Z", + "id": 6 + }, + { + "canary": "pieced", + "flow_document": "{\"_meta\":{\"uuid\":\"134a1456-20e1-11ee-990b-ffd12dfcd47f\"},\"canary\":\"pieced\",\"id\":7}", + "flow_published_at": "2023-07-12T18:22:35.841647Z", + "id": 7 + }, + { + "canary": "roaches", + "flow_document": "{\"_meta\":{\"uuid\":\"1b953992-20e1-11ee-990b-ffd12dfcd47f\"},\"canary\":\"roaches\",\"id\":8}", + "flow_published_at": "2023-07-12T18:22:49.755893Z", + "id": 8 + }, + { + "canary": "devilish", + "flow_document": "{\"_meta\":{\"uuid\":\"7e2df422-20e1-11ee-990b-ffd12dfcd47f\"},\"canary\":\"devilish\",\"id\":9}", + "flow_published_at": "2023-07-12T18:25:35.173533Z", + "id": 9 + }, + { + "canary": "glucose's", + "flow_document": "{\"_meta\":{\"uuid\":\"8458b580-20e1-11ee-990b-ffd12dfcd47f\"},\"canary\":\"glucose's\",\"id\":10}", + "flow_published_at": "2023-07-12T18:25:45.520064Z", + "id": 10 + } + ], "table": "`main`.`some-schema`.simple" } { - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"9afb3ff6-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":3,\"int\":8,\"str\":\"str 8\"}", - "flow_published_at": "2023-07-12 18:26:23.495167+00:00", - "id": 3, - "int": 8, - "str": "str 8" - }, - "table": "`main`.`some-schema`.duplicate_keys_standard" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"a1100a70-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":4,\"int\":9,\"str\":\"str 9\"}", - "flow_published_at": "2023-07-12 18:26:33.697752+00:00", - "id": 4, - "int": 9, - "str": "str 9" - }, - "table": "`main`.`some-schema`.duplicate_keys_standard" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"957348bc-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":2,\"int\":7,\"str\":\"str 7\"}", - "flow_published_at": "2023-07-12 18:26:14.215494+00:00", - "id": 2, - "int": 7, - "str": "str 7" - }, - "table": "`main`.`some-schema`.duplicate_keys_standard" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"a65203a8-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":5,\"int\":10,\"str\":\"str 10\"}", - "flow_published_at": "2023-07-12 18:26:42.518724+00:00", - "id": 5, - "int": 10, - "str": "str 10" - }, - "table": "`main`.`some-schema`.duplicate_keys_standard" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"8de85150-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":1,\"int\":6,\"str\":\"str 6\"}", - "flow_published_at": "2023-07-12 18:26:01.560712+00:00", - "id": 1, - "int": 6, - "str": "str 6" - }, + "rows": [ + { + "flow_document": "{\"_meta\":{\"uuid\":\"8de85150-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":1,\"int\":6,\"str\":\"str 6\"}", + "flow_published_at": "2023-07-12T18:26:01.560712Z", + "id": 1, + "int": 6, + "str": "str 6" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"957348bc-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":2,\"int\":7,\"str\":\"str 7\"}", + "flow_published_at": "2023-07-12T18:26:14.215494Z", + "id": 2, + "int": 7, + "str": "str 7" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"9afb3ff6-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":3,\"int\":8,\"str\":\"str 8\"}", + "flow_published_at": "2023-07-12T18:26:23.495167Z", + "id": 3, + "int": 8, + "str": "str 8" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"a1100a70-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":4,\"int\":9,\"str\":\"str 9\"}", + "flow_published_at": "2023-07-12T18:26:33.697752Z", + "id": 4, + "int": 9, + "str": "str 9" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"a65203a8-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":5,\"int\":10,\"str\":\"str 10\"}", + "flow_published_at": "2023-07-12T18:26:42.518724Z", + "id": 5, + "int": 10, + "str": "str 10" + } + ], "table": "`main`.`some-schema`.duplicate_keys_standard" } { - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"8de85150-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":1,\"int\":6,\"str\":\"str 6\"}", - "flow_published_at": "2023-07-12 18:26:01.560712+00:00", - "id": 1, - "int": 6, - "str": "str 6" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"957348bc-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":2,\"int\":7,\"str\":\"str 7\"}", - "flow_published_at": "2023-07-12 18:26:14.215494+00:00", - "id": 2, - "int": 7, - "str": "str 7" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"9afb3ff6-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":3,\"int\":8,\"str\":\"str 8\"}", - "flow_published_at": "2023-07-12 18:26:23.495167+00:00", - "id": 3, - "int": 8, - "str": "str 8" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"a1100a70-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":4,\"int\":9,\"str\":\"str 9\"}", - "flow_published_at": "2023-07-12 18:26:33.697752+00:00", - "id": 4, - "int": 9, - "str": "str 9" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"a65203a8-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":5,\"int\":10,\"str\":\"str 10\"}", - "flow_published_at": "2023-07-12 18:26:42.518724+00:00", - "id": 5, - "int": 10, - "str": "str 10" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"75c06bd6-20e0-11ee-990b-ffd12dfcd47f\"},\"id\":1,\"int\":1,\"str\":\"str 1\"}", - "flow_published_at": "2023-07-12 18:18:11.537199+00:00", - "id": 1, - "int": 1, - "str": "str 1" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"7dbe8ebc-20e0-11ee-990b-ffd12dfcd47f\"},\"id\":2,\"int\":2,\"str\":\"str 2\"}", - "flow_published_at": "2023-07-12 18:18:24.946758+00:00", - "id": 2, - "int": 2, - "str": "str 2" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"8bbf898a-20e0-11ee-990b-ffd12dfcd47f\"},\"id\":3,\"int\":3,\"str\":\"str 3\"}", - "flow_published_at": "2023-07-12 18:18:48.441281+00:00", - "id": 3, - "int": 3, - "str": "str 3" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"9b994ae4-20e0-11ee-990b-ffd12dfcd47f\"},\"id\":4,\"int\":4,\"str\":\"str 4\"}", - "flow_published_at": "2023-07-12 18:19:15.034186+00:00", - "id": 4, - "int": 4, - "str": "str 4" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta" -} -{ - "row": { - "flow_document": "{\"_meta\":{\"uuid\":\"bcef4bc6-20e0-11ee-990b-ffd12dfcd47f\"},\"id\":5,\"int\":5,\"str\":\"str 5\"}", - "flow_published_at": "2023-07-12 18:20:10.962631+00:00", - "id": 5, - "int": 5, - "str": "str 5" - }, + "rows": [ + { + "flow_document": "{\"_meta\":{\"uuid\":\"75c06bd6-20e0-11ee-990b-ffd12dfcd47f\"},\"id\":1,\"int\":1,\"str\":\"str 1\"}", + "flow_published_at": "2023-07-12T18:18:11.537199Z", + "id": 1, + "int": 1, + "str": "str 1" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"8de85150-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":1,\"int\":6,\"str\":\"str 6\"}", + "flow_published_at": "2023-07-12T18:26:01.560712Z", + "id": 1, + "int": 6, + "str": "str 6" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"7dbe8ebc-20e0-11ee-990b-ffd12dfcd47f\"},\"id\":2,\"int\":2,\"str\":\"str 2\"}", + "flow_published_at": "2023-07-12T18:18:24.946758Z", + "id": 2, + "int": 2, + "str": "str 2" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"957348bc-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":2,\"int\":7,\"str\":\"str 7\"}", + "flow_published_at": "2023-07-12T18:26:14.215494Z", + "id": 2, + "int": 7, + "str": "str 7" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"8bbf898a-20e0-11ee-990b-ffd12dfcd47f\"},\"id\":3,\"int\":3,\"str\":\"str 3\"}", + "flow_published_at": "2023-07-12T18:18:48.441281Z", + "id": 3, + "int": 3, + "str": "str 3" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"9afb3ff6-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":3,\"int\":8,\"str\":\"str 8\"}", + "flow_published_at": "2023-07-12T18:26:23.495167Z", + "id": 3, + "int": 8, + "str": "str 8" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"9b994ae4-20e0-11ee-990b-ffd12dfcd47f\"},\"id\":4,\"int\":4,\"str\":\"str 4\"}", + "flow_published_at": "2023-07-12T18:19:15.034186Z", + "id": 4, + "int": 4, + "str": "str 4" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"a1100a70-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":4,\"int\":9,\"str\":\"str 9\"}", + "flow_published_at": "2023-07-12T18:26:33.697752Z", + "id": 4, + "int": 9, + "str": "str 9" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"bcef4bc6-20e0-11ee-990b-ffd12dfcd47f\"},\"id\":5,\"int\":5,\"str\":\"str 5\"}", + "flow_published_at": "2023-07-12T18:20:10.962631Z", + "id": 5, + "int": 5, + "str": "str 5" + }, + { + "flow_document": "{\"_meta\":{\"uuid\":\"a65203a8-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":5,\"int\":10,\"str\":\"str 10\"}", + "flow_published_at": "2023-07-12T18:26:42.518724Z", + "id": 5, + "int": 10, + "str": "str 10" + } + ], "table": "`main`.`some-schema`.duplicate_keys_delta" } { - "row": { - "flow_published_at": "2023-07-12 18:26:01.560712+00:00", - "id": 1, - "int": 6, - "str": "str 6" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta_exclude_flow_doc" -} -{ - "row": { - "flow_published_at": "2023-07-12 18:26:14.215494+00:00", - "id": 2, - "int": 7, - "str": "str 7" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta_exclude_flow_doc" -} -{ - "row": { - "flow_published_at": "2023-07-12 18:26:23.495167+00:00", - "id": 3, - "int": 8, - "str": "str 8" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta_exclude_flow_doc" -} -{ - "row": { - "flow_published_at": "2023-07-12 18:26:33.697752+00:00", - "id": 4, - "int": 9, - "str": "str 9" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta_exclude_flow_doc" -} -{ - "row": { - "flow_published_at": "2023-07-12 18:26:42.518724+00:00", - "id": 5, - "int": 10, - "str": "str 10" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta_exclude_flow_doc" -} -{ - "row": { - "flow_published_at": "2023-07-12 18:18:11.537199+00:00", - "id": 1, - "int": 1, - "str": "str 1" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta_exclude_flow_doc" -} -{ - "row": { - "flow_published_at": "2023-07-12 18:18:24.946758+00:00", - "id": 2, - "int": 2, - "str": "str 2" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta_exclude_flow_doc" -} -{ - "row": { - "flow_published_at": "2023-07-12 18:18:48.441281+00:00", - "id": 3, - "int": 3, - "str": "str 3" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta_exclude_flow_doc" -} -{ - "row": { - "flow_published_at": "2023-07-12 18:19:15.034186+00:00", - "id": 4, - "int": 4, - "str": "str 4" - }, - "table": "`main`.`some-schema`.duplicate_keys_delta_exclude_flow_doc" -} -{ - "row": { - "flow_published_at": "2023-07-12 18:20:10.962631+00:00", - "id": 5, - "int": 5, - "str": "str 5" - }, + "rows": [ + { + "flow_published_at": "2023-07-12T18:18:11.537199Z", + "id": 1, + "int": 1, + "str": "str 1" + }, + { + "flow_published_at": "2023-07-12T18:26:01.560712Z", + "id": 1, + "int": 6, + "str": "str 6" + }, + { + "flow_published_at": "2023-07-12T18:18:24.946758Z", + "id": 2, + "int": 2, + "str": "str 2" + }, + { + "flow_published_at": "2023-07-12T18:26:14.215494Z", + "id": 2, + "int": 7, + "str": "str 7" + }, + { + "flow_published_at": "2023-07-12T18:18:48.441281Z", + "id": 3, + "int": 3, + "str": "str 3" + }, + { + "flow_published_at": "2023-07-12T18:26:23.495167Z", + "id": 3, + "int": 8, + "str": "str 8" + }, + { + "flow_published_at": "2023-07-12T18:19:15.034186Z", + "id": 4, + "int": 4, + "str": "str 4" + }, + { + "flow_published_at": "2023-07-12T18:26:33.697752Z", + "id": 4, + "int": 9, + "str": "str 9" + }, + { + "flow_published_at": "2023-07-12T18:20:10.962631Z", + "id": 5, + "int": 5, + "str": "str 5" + }, + { + "flow_published_at": "2023-07-12T18:26:42.518724Z", + "id": 5, + "int": 10, + "str": "str 10" + } + ], "table": "`main`.`some-schema`.duplicate_keys_delta_exclude_flow_doc" } { - "row": { - "array_int": "[11,12]", - "bool_field": "False", - "float_field": 1.1, - "flow_document": "{\"_meta\":{\"uuid\":\"3fd2ec78-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[11,12],\"bool_field\":false,\"float_field\":1.1,\"id\":1,\"multiple\":1,\"nested\":{\"id\":\"i1\"},\"nullable_int\":null,\"str_field\":\"str1\"}", - "flow_published_at": "2023-07-12 18:23:50.558220+00:00", - "id": 1, - "multiple": 1, - "nested": "{\"id\":\"i1\"}", - "nullable_int": "", - "str_field": "str1" - }, - "table": "`main`.`some-schema`.multiple_types" -} -{ - "row": { - "array_int": "[21,22]", - "bool_field": "True", - "float_field": 2.2, - "flow_document": "{\"_meta\":{\"uuid\":\"4b37f0b8-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[21,22],\"bool_field\":true,\"float_field\":2.2,\"id\":2,\"multiple\":2.2,\"nested\":{\"id\":\"i2\"},\"nullable_int\":2,\"str_field\":\"str2\"}", - "flow_published_at": "2023-07-12 18:24:09.675180+00:00", - "id": 2, - "multiple": 2.2, - "nested": "{\"id\":\"i2\"}", - "nullable_int": 2, - "str_field": "str2" - }, - "table": "`main`.`some-schema`.multiple_types" -} -{ - "row": { - "array_int": "[31,32]", - "bool_field": "False", - "float_field": 3.3, - "flow_document": "{\"_meta\":{\"uuid\":\"51016380-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[31,32],\"bool_field\":false,\"float_field\":3.3,\"id\":3,\"multiple\":true,\"nested\":{\"id\":\"i3\"},\"nullable_int\":null,\"str_field\":\"str3\"}", - "flow_published_at": "2023-07-12 18:24:19.384000+00:00", - "id": 3, - "multiple": "true", - "nested": "{\"id\":\"i3\"}", - "nullable_int": "", - "str_field": "str3" - }, - "table": "`main`.`some-schema`.multiple_types" -} -{ - "row": { - "array_int": "[41,42]", - "bool_field": "True", - "float_field": 4.4, - "flow_document": "{\"_meta\":{\"uuid\":\"5660aaca-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[41,42],\"bool_field\":true,\"float_field\":4.4,\"id\":4,\"multiple\":false,\"nested\":{\"id\":\"i4\"},\"nullable_int\":4,\"str_field\":\"str4\"}", - "flow_published_at": "2023-07-12 18:24:28.397025+00:00", - "id": 4, - "multiple": "false", - "nested": "{\"id\":\"i4\"}", - "nullable_int": 4, - "str_field": "str4" - }, - "table": "`main`.`some-schema`.multiple_types" -} -{ - "row": { - "array_int": "[51,52]", - "bool_field": "False", - "float_field": 5.5, - "flow_document": "{\"_meta\":{\"uuid\":\"5af9e236-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[51,52],\"bool_field\":false,\"float_field\":5.5,\"id\":5,\"multiple\":\"string five\",\"nested\":{\"id\":\"i5\"},\"nullable_int\":null,\"str_field\":\"str5\"}", - "flow_published_at": "2023-07-12 18:24:36.112031+00:00", - "id": 5, - "multiple": "\"string five\"", - "nested": "{\"id\":\"i5\"}", - "nullable_int": "", - "str_field": "str5" - }, - "table": "`main`.`some-schema`.multiple_types" -} -{ - "row": { - "array_int": "[91,92]", - "bool_field": "False", - "float_field": 99.99, - "flow_document": "{\"_meta\":{\"uuid\":\"f5b064a8-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[91,92],\"bool_field\":false,\"float_field\":99.99,\"id\":9,\"nested\":{\"id\":\"i9\"},\"nullable_int\":null,\"str_field\":\"str9 v2\"}", - "flow_published_at": "2023-07-12 18:28:55.677252+00:00", - "id": 9, - "multiple": "", - "nested": "{\"id\":\"i9\"}", - "nullable_int": "", - "str_field": "str9 v2" - }, - "table": "`main`.`some-schema`.multiple_types" -} -{ - "row": { - "array_int": "[61,62]", - "bool_field": "True", - "float_field": 66.66, - "flow_document": "{\"_meta\":{\"uuid\":\"e4b646c2-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[61,62],\"bool_field\":true,\"float_field\":66.66,\"id\":6,\"multiple\":[\"one\",2,true],\"nested\":{\"id\":\"i6\"},\"nullable_int\":6,\"str_field\":\"str6 v2\"}", - "flow_published_at": "2023-07-12 18:28:27.194541+00:00", - "id": 6, - "multiple": "[\"one\",2,true]", - "nested": "{\"id\":\"i6\"}", - "nullable_int": 6, - "str_field": "str6 v2" - }, - "table": "`main`.`some-schema`.multiple_types" -} -{ - "row": { - "array_int": "[71,72]", - "bool_field": "False", - "float_field": 77.77, - "flow_document": "{\"_meta\":{\"uuid\":\"eb40dafc-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[71,72],\"bool_field\":false,\"float_field\":77.77,\"id\":7,\"multiple\":{\"object\":\"seven\"},\"nested\":{\"id\":\"i7\"},\"nullable_int\":null,\"str_field\":\"str7 v2\"}", - "flow_published_at": "2023-07-12 18:28:38.169062+00:00", - "id": 7, - "multiple": "{\"object\":\"seven\"}", - "nested": "{\"id\":\"i7\"}", - "nullable_int": "", - "str_field": "str7 v2" - }, - "table": "`main`.`some-schema`.multiple_types" -} -{ - "row": { - "array_int": "[81,82]", - "bool_field": "True", - "float_field": 88.88, - "flow_document": "{\"_meta\":{\"uuid\":\"f163eef6-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[81,82],\"bool_field\":true,\"float_field\":88.88,\"id\":8,\"multiple\":null,\"nested\":{\"id\":\"i8\"},\"nullable_int\":8,\"str_field\":\"str8 v2\"}", - "flow_published_at": "2023-07-12 18:28:48.465279+00:00", - "id": 8, - "multiple": "", - "nested": "{\"id\":\"i8\"}", - "nullable_int": 8, - "str_field": "str8 v2" - }, - "table": "`main`.`some-schema`.multiple_types" -} -{ - "row": { - "array_int": "[1,2]", - "bool_field": "True", - "float_field": 1010.101, - "flow_document": "{\"_meta\":{\"uuid\":\"fb0cc8b0-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[1,2],\"bool_field\":true,\"float_field\":1010.101,\"id\":10,\"nested\":{\"id\":\"i10\"},\"nullable_int\":10,\"str_field\":\"str10 v2\"}", - "flow_published_at": "2023-07-12 18:29:04.671352+00:00", - "id": 10, - "multiple": "", - "nested": "{\"id\":\"i10\"}", - "nullable_int": 10, - "str_field": "str10 v2" - }, + "rows": [ + { + "array_int": "[11,12]", + "bool_field": false, + "float_field": 1.1, + "flow_document": "{\"_meta\":{\"uuid\":\"3fd2ec78-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[11,12],\"bool_field\":false,\"float_field\":1.1,\"id\":1,\"multiple\":1,\"nested\":{\"id\":\"i1\"},\"nullable_int\":null,\"str_field\":\"str1\"}", + "flow_published_at": "2023-07-12T18:23:50.55822Z", + "id": 1, + "multiple": "1", + "nested": "{\"id\":\"i1\"}", + "nullable_int": null, + "str_field": "str1" + }, + { + "array_int": "[21,22]", + "bool_field": true, + "float_field": 2.2, + "flow_document": "{\"_meta\":{\"uuid\":\"4b37f0b8-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[21,22],\"bool_field\":true,\"float_field\":2.2,\"id\":2,\"multiple\":2.2,\"nested\":{\"id\":\"i2\"},\"nullable_int\":2,\"str_field\":\"str2\"}", + "flow_published_at": "2023-07-12T18:24:09.67518Z", + "id": 2, + "multiple": "2.2", + "nested": "{\"id\":\"i2\"}", + "nullable_int": 2, + "str_field": "str2" + }, + { + "array_int": "[31,32]", + "bool_field": false, + "float_field": 3.3, + "flow_document": "{\"_meta\":{\"uuid\":\"51016380-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[31,32],\"bool_field\":false,\"float_field\":3.3,\"id\":3,\"multiple\":true,\"nested\":{\"id\":\"i3\"},\"nullable_int\":null,\"str_field\":\"str3\"}", + "flow_published_at": "2023-07-12T18:24:19.384Z", + "id": 3, + "multiple": "true", + "nested": "{\"id\":\"i3\"}", + "nullable_int": null, + "str_field": "str3" + }, + { + "array_int": "[41,42]", + "bool_field": true, + "float_field": 4.4, + "flow_document": "{\"_meta\":{\"uuid\":\"5660aaca-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[41,42],\"bool_field\":true,\"float_field\":4.4,\"id\":4,\"multiple\":false,\"nested\":{\"id\":\"i4\"},\"nullable_int\":4,\"str_field\":\"str4\"}", + "flow_published_at": "2023-07-12T18:24:28.397025Z", + "id": 4, + "multiple": "false", + "nested": "{\"id\":\"i4\"}", + "nullable_int": 4, + "str_field": "str4" + }, + { + "array_int": "[51,52]", + "bool_field": false, + "float_field": 5.5, + "flow_document": "{\"_meta\":{\"uuid\":\"5af9e236-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[51,52],\"bool_field\":false,\"float_field\":5.5,\"id\":5,\"multiple\":\"string five\",\"nested\":{\"id\":\"i5\"},\"nullable_int\":null,\"str_field\":\"str5\"}", + "flow_published_at": "2023-07-12T18:24:36.112031Z", + "id": 5, + "multiple": "\"string five\"", + "nested": "{\"id\":\"i5\"}", + "nullable_int": null, + "str_field": "str5" + }, + { + "array_int": "[61,62]", + "bool_field": true, + "float_field": 66.66, + "flow_document": "{\"_meta\":{\"uuid\":\"e4b646c2-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[61,62],\"bool_field\":true,\"float_field\":66.66,\"id\":6,\"multiple\":[\"one\",2,true],\"nested\":{\"id\":\"i6\"},\"nullable_int\":6,\"str_field\":\"str6 v2\"}", + "flow_published_at": "2023-07-12T18:28:27.194541Z", + "id": 6, + "multiple": "[\"one\",2,true]", + "nested": "{\"id\":\"i6\"}", + "nullable_int": 6, + "str_field": "str6 v2" + }, + { + "array_int": "[71,72]", + "bool_field": false, + "float_field": 77.77, + "flow_document": "{\"_meta\":{\"uuid\":\"eb40dafc-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[71,72],\"bool_field\":false,\"float_field\":77.77,\"id\":7,\"multiple\":{\"object\":\"seven\"},\"nested\":{\"id\":\"i7\"},\"nullable_int\":null,\"str_field\":\"str7 v2\"}", + "flow_published_at": "2023-07-12T18:28:38.169062Z", + "id": 7, + "multiple": "{\"object\":\"seven\"}", + "nested": "{\"id\":\"i7\"}", + "nullable_int": null, + "str_field": "str7 v2" + }, + { + "array_int": "[81,82]", + "bool_field": true, + "float_field": 88.88, + "flow_document": "{\"_meta\":{\"uuid\":\"f163eef6-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[81,82],\"bool_field\":true,\"float_field\":88.88,\"id\":8,\"multiple\":null,\"nested\":{\"id\":\"i8\"},\"nullable_int\":8,\"str_field\":\"str8 v2\"}", + "flow_published_at": "2023-07-12T18:28:48.465279Z", + "id": 8, + "multiple": "", + "nested": "{\"id\":\"i8\"}", + "nullable_int": 8, + "str_field": "str8 v2" + }, + { + "array_int": "[91,92]", + "bool_field": false, + "float_field": 99.99, + "flow_document": "{\"_meta\":{\"uuid\":\"f5b064a8-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[91,92],\"bool_field\":false,\"float_field\":99.99,\"id\":9,\"nested\":{\"id\":\"i9\"},\"nullable_int\":null,\"str_field\":\"str9 v2\"}", + "flow_published_at": "2023-07-12T18:28:55.677252Z", + "id": 9, + "multiple": "", + "nested": "{\"id\":\"i9\"}", + "nullable_int": null, + "str_field": "str9 v2" + }, + { + "array_int": "[1,2]", + "bool_field": true, + "float_field": 1010.101, + "flow_document": "{\"_meta\":{\"uuid\":\"fb0cc8b0-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[1,2],\"bool_field\":true,\"float_field\":1010.101,\"id\":10,\"nested\":{\"id\":\"i10\"},\"nullable_int\":10,\"str_field\":\"str10 v2\"}", + "flow_published_at": "2023-07-12T18:29:04.671352Z", + "id": 10, + "multiple": "", + "nested": "{\"id\":\"i10\"}", + "nullable_int": 10, + "str_field": "str10 v2" + } + ], "table": "`main`.`some-schema`.multiple_types" } { - "row": { - "date": "1000-03-03", - "datetime": "1000-03-03 23:59:38.1", - "flow_published_at": "2023-07-12 18:18:11.537199+00:00", - "id": 3, - "int_and_str": 3, - "int_str": 30, - "num_and_str": 3.1, - "num_str": 30.1, - "time": "23:59:38.10Z" - }, - "table": "`main`.`some-schema`.formatted_strings" -} -{ - "row": { - "date": "2023-08-29", - "datetime": "2023-08-29 23:59:38", - "flow_published_at": "2023-07-12 18:18:48.441281+00:00", - "id": 4, - "int_and_str": 4, - "int_str": 40, - "num_and_str": 4.1, - "num_str": 40.1, - "time": "23:59:38Z" - }, - "table": "`main`.`some-schema`.formatted_strings" -} -{ - "row": { - "date": "0000-01-01", - "datetime": "0000-01-01 00:00:00", - "flow_published_at": "2023-07-12 18:27:01.912219+00:00", - "id": 1, - "int_and_str": 1, - "int_str": 10, - "num_and_str": 1.1, - "num_str": 10.1, - "time": "00:00:00Z" - }, - "table": "`main`.`some-schema`.formatted_strings" -} -{ - "row": { - "date": "1999-02-02", - "datetime": "1999-02-02 14:20:12.33", - "flow_published_at": "2023-07-12 18:27:12.354610+00:00", - "id": 2, - "int_and_str": 2, - "int_str": 20, - "num_and_str": 2.1, - "num_str": 20.1, - "time": "14:20:12.33Z" - }, - "table": "`main`.`some-schema`.formatted_strings" -} -{ - "row": { - "date": "9999-12-31", - "datetime": "9999-12-31 23:59:59", - "flow_published_at": "2023-07-12 18:27:25.889321+00:00", - "id": 5, - "int_and_str": 5, - "int_str": 50, - "num_and_str": 5.1, - "num_str": 50.1, - "time": "23:59:59Z" - }, - "table": "`main`.`some-schema`.formatted_strings" -} -{ - "row": { - "date": "", - "datetime": "", - "flow_published_at": "2023-07-12 18:27:25.889321+00:00", - "id": 8, - "int_and_str": "", - "int_str": "", - "num_and_str": "", - "num_str": "", - "time": "" - }, - "table": "`main`.`some-schema`.formatted_strings" -} -{ - "row": { - "date": "", - "datetime": "", - "flow_published_at": "2023-07-12 18:27:25.889321+00:00", - "id": 9, - "int_and_str": "", - "int_str": "", - "num_and_str": "", - "num_str": "inf", - "time": "" - }, - "table": "`main`.`some-schema`.formatted_strings" -} -{ - "row": { - "date": "", - "datetime": "", - "flow_published_at": "2023-07-12 18:27:25.889321+00:00", - "id": 10, - "int_and_str": "", - "int_str": "", - "num_and_str": "", - "num_str": "-inf", - "time": "" - }, + "rows": [ + { + "date": "0000-01-01T00:00:00Z", + "datetime": "0000-01-01T00:00:00Z", + "flow_document": "{\"_meta\":{\"uuid\":\"b1e13a0e-20e1-11ee-990b-ffd12dfcd47f\"},\"date\":\"0000-01-01\",\"datetime\":\"0000-01-01T00:00:00Z\",\"id\":1,\"int_and_str\":1,\"int_str\":\"10\",\"num_and_str\":1.1,\"num_str\":\"10.1\",\"time\":\"00:00:00Z\"}", + "flow_published_at": "2023-07-12T18:27:01.912219Z", + "id": 1, + "int_and_str": 1, + "int_str": 10, + "num_and_str": 1.1, + "num_str": 10.1, + "time": "00:00:00Z" + }, + { + "date": "1999-02-02T00:00:00Z", + "datetime": "1999-02-02T14:20:12.33Z", + "flow_document": "{\"_meta\":{\"uuid\":\"b81a9bf4-20e1-11ee-990b-ffd12dfcd47f\"},\"date\":\"1999-02-02\",\"datetime\":\"1999-02-02T14:20:12.33Z\",\"id\":2,\"int_and_str\":2,\"int_str\":\"20\",\"num_and_str\":2.1,\"num_str\":\"20.1\",\"time\":\"14:20:12.33Z\"}", + "flow_published_at": "2023-07-12T18:27:12.35461Z", + "id": 2, + "int_and_str": 2, + "int_str": 20, + "num_and_str": 2.1, + "num_str": 20.1, + "time": "14:20:12.33Z" + }, + { + "date": "1000-03-03T00:00:00Z", + "datetime": "1000-03-03T23:59:38.1Z", + "flow_document": "{\"_meta\":{\"uuid\":\"75c06bd6-20e0-11ee-990b-ffd12dfcd47f\"},\"date\":\"1000-03-03\",\"datetime\":\"1000-03-03T23:59:38.10Z\",\"id\":3,\"int_and_str\":3,\"int_str\":\"30\",\"num_and_str\":3.1,\"num_str\":\"30.1\",\"time\":\"23:59:38.10Z\"}", + "flow_published_at": "2023-07-12T18:18:11.537199Z", + "id": 3, + "int_and_str": 3, + "int_str": 30, + "num_and_str": 3.1, + "num_str": 30.1, + "time": "23:59:38.10Z" + }, + { + "date": "2023-08-29T00:00:00Z", + "datetime": "2023-08-29T23:59:38Z", + "flow_document": "{\"_meta\":{\"uuid\":\"8bbf898a-20e0-11ee-990b-ffd12dfcd47f\"},\"date\":\"2023-08-29\",\"datetime\":\"2023-08-29T23:59:38Z\",\"id\":4,\"int_and_str\":\"4\",\"int_str\":\"40\",\"num_and_str\":\"4.1\",\"num_str\":\"40.1\",\"time\":\"23:59:38Z\"}", + "flow_published_at": "2023-07-12T18:18:48.441281Z", + "id": 4, + "int_and_str": 4, + "int_str": 40, + "num_and_str": 4.1, + "num_str": 40.1, + "time": "23:59:38Z" + }, + { + "date": "9999-12-31T00:00:00Z", + "datetime": "9999-12-31T23:59:59Z", + "flow_document": "{\"_meta\":{\"uuid\":\"c02bd79a-20e1-11ee-990b-ffd12dfcd47f\"},\"date\":\"9999-12-31\",\"datetime\":\"9999-12-31T23:59:59Z\",\"id\":5,\"int_and_str\":\"5\",\"int_str\":\"50\",\"num_and_str\":\"5.1\",\"num_str\":\"50.1\",\"time\":\"23:59:59Z\"}", + "flow_published_at": "2023-07-12T18:27:25.889321Z", + "id": 5, + "int_and_str": 5, + "int_str": 50, + "num_and_str": 5.1, + "num_str": 50.1, + "time": "23:59:59Z" + }, + { + "date": null, + "datetime": null, + "flow_document": "{\"_meta\":{\"uuid\":\"c02bd79a-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":8,\"num_str\":\"NaN\"}", + "flow_published_at": "2023-07-12T18:27:25.889321Z", + "id": 8, + "int_and_str": null, + "int_str": null, + "num_and_str": null, + "num_str": "NaN", + "time": null + }, + { + "date": null, + "datetime": null, + "flow_document": "{\"_meta\":{\"uuid\":\"c02bd79a-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":9,\"num_str\":\"Infinity\"}", + "flow_published_at": "2023-07-12T18:27:25.889321Z", + "id": 9, + "int_and_str": null, + "int_str": null, + "num_and_str": null, + "num_str": "Infinity", + "time": null + }, + { + "date": null, + "datetime": null, + "flow_document": "{\"_meta\":{\"uuid\":\"c02bd79a-20e1-11ee-990b-ffd12dfcd47f\"},\"id\":10,\"num_str\":\"-Infinity\"}", + "flow_published_at": "2023-07-12T18:27:25.889321Z", + "id": 10, + "int_and_str": null, + "int_str": null, + "num_and_str": null, + "num_str": "-Infinity", + "time": null + } + ], "table": "`main`.`some-schema`.formatted_strings" } { From 2b40a89f0351a7481fecee5aa0d5286a973189aa Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 20 Mar 2024 15:47:11 +0000 Subject: [PATCH 33/96] materialize-databricks: materialize JSON null as null --- materialize-databricks/sqlgen.go | 2 +- tests/materialize/materialize-databricks/snapshot.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/materialize-databricks/sqlgen.go b/materialize-databricks/sqlgen.go index 5f16c8a69c..94691b5396 100644 --- a/materialize-databricks/sqlgen.go +++ b/materialize-databricks/sqlgen.go @@ -29,7 +29,7 @@ var jsonConverter sql.ElementConverter = func(te tuple.TupleElement) (interface{ case json.RawMessage: return string(ii), nil case nil: - return string(json.RawMessage(nil)), nil + return nil, nil default: var m, err = json.Marshal(te) if err != nil { diff --git a/tests/materialize/materialize-databricks/snapshot.json b/tests/materialize/materialize-databricks/snapshot.json index 0e05ddc37e..10ff2e680e 100644 --- a/tests/materialize/materialize-databricks/snapshot.json +++ b/tests/materialize/materialize-databricks/snapshot.json @@ -667,7 +667,7 @@ "flow_document": "{\"_meta\":{\"uuid\":\"f163eef6-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[81,82],\"bool_field\":true,\"float_field\":88.88,\"id\":8,\"multiple\":null,\"nested\":{\"id\":\"i8\"},\"nullable_int\":8,\"str_field\":\"str8 v2\"}", "flow_published_at": "2023-07-12T18:28:48.465279Z", "id": 8, - "multiple": "", + "multiple": null, "nested": "{\"id\":\"i8\"}", "nullable_int": 8, "str_field": "str8 v2" @@ -679,7 +679,7 @@ "flow_document": "{\"_meta\":{\"uuid\":\"f5b064a8-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[91,92],\"bool_field\":false,\"float_field\":99.99,\"id\":9,\"nested\":{\"id\":\"i9\"},\"nullable_int\":null,\"str_field\":\"str9 v2\"}", "flow_published_at": "2023-07-12T18:28:55.677252Z", "id": 9, - "multiple": "", + "multiple": null, "nested": "{\"id\":\"i9\"}", "nullable_int": null, "str_field": "str9 v2" @@ -691,7 +691,7 @@ "flow_document": "{\"_meta\":{\"uuid\":\"fb0cc8b0-20e1-11ee-990b-ffd12dfcd47f\"},\"array_int\":[1,2],\"bool_field\":true,\"float_field\":1010.101,\"id\":10,\"nested\":{\"id\":\"i10\"},\"nullable_int\":10,\"str_field\":\"str10 v2\"}", "flow_published_at": "2023-07-12T18:29:04.671352Z", "id": 10, - "multiple": "", + "multiple": null, "nested": "{\"id\":\"i10\"}", "nullable_int": 10, "str_field": "str10 v2" From 3e1b22363b612697e91bda7f7edf4b748e89fd58 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Wed, 20 Mar 2024 21:08:47 -0500 Subject: [PATCH 34/96] sqlcapture: Use segmentio JSON encoder instead of stdlib JSON encoding is one of the single largest bottlenecks in the SQL CDC captures at present. A recent example in production where replication throughput was constrained by connector CPU had JSON serialization at around 70% of main thread CPU usage. The standard library `encoding/json` package is fairly slow with the way it uses reflection everywhere, and the package `github.com/segmentio/encoding/json` is a drop-in replacement which performs 2-3x better in most benchmarks. So swapping out the JSON encoder used here is a very easy way to improve the situation, although in the long run there are fancier approaches we could potentially use to make it even faster. See also: - source-mysql-batch in https://github.com/estuary/connectors/pull/1005 - materialize-sql in https://github.com/estuary/connectors/pull/1010 --- sqlcapture/capture.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlcapture/capture.go b/sqlcapture/capture.go index 3c6b525a7d..ad75149b1d 100644 --- a/sqlcapture/capture.go +++ b/sqlcapture/capture.go @@ -2,7 +2,6 @@ package sqlcapture import ( "context" - "encoding/json" "fmt" "io" "math/rand" @@ -11,6 +10,8 @@ import ( "sync" "time" + "github.com/segmentio/encoding/json" + boilerplate "github.com/estuary/connectors/source-boilerplate" pf "github.com/estuary/flow/go/protocols/flow" "github.com/google/uuid" From 0ad947da76befdc4e26992a0ecbc44934268393f Mon Sep 17 00:00:00 2001 From: Joseph Shearer Date: Wed, 20 Mar 2024 16:24:55 -0400 Subject: [PATCH 35/96] cdk: Fix correctly escape fields coming from Airbyte --- estuary-cdk/estuary_cdk/shim_airbyte_cdk.py | 47 +++++++++++++++++---- estuary-cdk/tests/__init__.py | 0 estuary-cdk/tests/test_airbyte_cdk.py | 10 +++++ 3 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 estuary-cdk/tests/__init__.py create mode 100644 estuary-cdk/tests/test_airbyte_cdk.py diff --git a/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py b/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py index d96eecde98..c4016f1442 100644 --- a/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py +++ b/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from logging import Logger from pydantic import Field -from typing import Any, ClassVar, Annotated, Callable, Awaitable, Literal +from typing import Any, ClassVar, Annotated, Callable, Awaitable, List, Literal import logging import os @@ -86,6 +86,36 @@ class Document(common.BaseDocument, extra="allow"): pass +def escape_field(field: str) -> str: + return field.replace("~", "~0").replace("/", "~1") + + +def transform_airbyte_key(key: str | List[str] | List[List[str]]) -> List[str]: + key_fields: List[str] = [] + + if isinstance(key, str): + # key = "piz/za" + # key_fields: ["piz~1za"] + key_fields = [escape_field(key)] + elif isinstance(key, list): + for component in key: + if isinstance(component, str): + # key = ["piz/za", "par~ty"] + # key_fields: ["piz~1za", "pa~0rty"] + key_fields.append(escape_field(component)) + elif isinstance(component, list): + # key = [["pizza", "che/ese"], "potato"] + # Implies a document like {"potato": 12, "pizza": {"che/ese": 5}} + # key_fields: ["pizza/che~1ese", "potato"] + key_fields.append( + "/".join((escape_field(field) for field in component)) + ) + else: + raise ValueError(f"Invalid key component: {component}") + + return key_fields + + @dataclass class CaptureShim(BaseCaptureConnector): delegate: AirbyteSource @@ -124,11 +154,8 @@ async def _all_resources( if stream.source_defined_primary_key: # Map array of array of property names into an array of JSON pointers. key = [ - "/" - + "/".join( - p.replace("~", "~0").replace("/", "~1") for p in component - ) - for component in stream.source_defined_primary_key + "/" + key + for key in transform_airbyte_key(stream.source_defined_primary_key) ] elif resource_config.sync_mode == "full_refresh": # Synthesize a key based on the record's order within each stream refresh. @@ -176,13 +203,17 @@ async def spec(self, log: Logger, _: request.Spec) -> ConnectorSpec: ) async def discover( - self, log: Logger, discover: request.Discover[EndpointConfig], + self, + log: Logger, + discover: request.Discover[EndpointConfig], ) -> response.Discovered: resources = await self._all_resources(log, discover.config) return common.discovered(resources) async def validate( - self, log: Logger, validate: request.Validate[EndpointConfig, ResourceConfig], + self, + log: Logger, + validate: request.Validate[EndpointConfig, ResourceConfig], ) -> response.Validated: result = self.delegate.check(log, validate.config) diff --git a/estuary-cdk/tests/__init__.py b/estuary-cdk/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/estuary-cdk/tests/test_airbyte_cdk.py b/estuary-cdk/tests/test_airbyte_cdk.py new file mode 100644 index 0000000000..5b2c1348a3 --- /dev/null +++ b/estuary-cdk/tests/test_airbyte_cdk.py @@ -0,0 +1,10 @@ +from estuary_cdk.shim_airbyte_cdk import transform_airbyte_key + +def test_transform_airbyte_key(): + assert transform_airbyte_key("pizza") == ["pizza"] + assert transform_airbyte_key("piz/za") == ["piz~1za"] + assert transform_airbyte_key(["piz/za"]) == ["piz~1za"] + assert transform_airbyte_key(["piz/za", "par~ty"]) == ["piz~1za", "par~0ty"] + assert transform_airbyte_key([["pizza", "che/ese"], "potato"]) == [ + "pizza/che~1ese", "potato" + ] From a4ef4bc9d57499d47a85e27f2fc9b20c75b13699 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Thu, 21 Mar 2024 14:04:17 -0500 Subject: [PATCH 36/96] source-redshift-batch: The most barebones README imaginable Just some notes for myself so this is easier if/when I have to set up another Redshift test instance in the future. This will also probably get copied over and cleaned up massively whenever I get around to writing some proper docs. --- source-redshift-batch/README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/source-redshift-batch/README.md b/source-redshift-batch/README.md index f50cc04db7..662e0e08ea 100644 --- a/source-redshift-batch/README.md +++ b/source-redshift-batch/README.md @@ -1,4 +1,20 @@ Redshift Batch Source Connector =============================== -TODO \ No newline at end of file +Redshift external connectivity setup: + +1. Set 'Publicly Accessible' to 'On' for the workgroup +2. The associated VPC Security Group needs an Inbound Rule permitting Redshift + traffic from `0.0.0.0/0` or from the Estuary public IP. +3. The associated VPC Route Table needs a route for traffic to `0.0.0.0/0` or + the Estuary public IP to egress via an Internet Gateway. +4. If the VPC uses network ACLs, they also need to permit inbound and outbound + Redshift traffic. + +Example user setup: + + $ psql 'postgres://admin:Secret1234@default-workgroup.123456789123.us-east-1.redshift-serverless.amazonaws.com:5439/dev' + dev=# CREATE USER flow_capture WITH PASSWORD 'Secret1234'; + dev=# CREATE SCHEMA test; + dev=# ALTER DEFAULT PRIVILEGES FOR USER admin IN SCHEMA test GRANT ALL ON TABLES TO flow_capture; + dev=# GRANT ALL ON SCHEMA test TO flow_capture; \ No newline at end of file From 03b074286f2bcbd781df744e282b051f4fbd02a1 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Thu, 21 Mar 2024 13:57:49 -0500 Subject: [PATCH 37/96] source-redshift-batch: Discover all columns and omit fallback key This commit modifies the `source-redshift-batch` connector so that the generated schemas for discovered collections will include all columns with appropriate JSON schema types, rather than just the primary-key columns as it did previously. In addition `source-redshift-batch` will no longer suggest a fallback collection key of `[/_meta/polled, /_meta/offset]` for tables without a primary key. This means that users will be forced to pick their desired collection key, which is why it was important that we adjust the discovered schema so that they have appropriate options to choose from. This fixes https://github.com/estuary/connectors/issues/1383 --- .../.snapshots/TestBasicCapture-Discovery | 6 +- .../.snapshots/TestBasicDatatypes-Discovery | 21 +- .../.snapshots/TestKeyDiscovery | 6 +- .../.snapshots/TestKeylessDiscovery | 73 ++++ .../.snapshots/TestSchemaFilter-FilteredIn | 6 +- .../.snapshots/TestSchemaFilter-Unfiltered | 6 +- source-redshift-batch/driver.go | 321 ++++++++++++------ source-redshift-batch/main_test.go | 16 +- 8 files changed, 348 insertions(+), 107 deletions(-) create mode 100644 source-redshift-batch/.snapshots/TestKeylessDiscovery diff --git a/source-redshift-batch/.snapshots/TestBasicCapture-Discovery b/source-redshift-batch/.snapshots/TestBasicCapture-Discovery index 205414c72c..ac8bbc7f1f 100644 --- a/source-redshift-batch/.snapshots/TestBasicCapture-Discovery +++ b/source-redshift-batch/.snapshots/TestBasicCapture-Discovery @@ -39,11 +39,13 @@ Binding 0: "index" ] }, + "data": { + "type": "string" + }, "id": { "type": "integer" } - }, - "x-infer-schema": true + } }, "key": [ "/id" diff --git a/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery b/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery index 3971e35c44..cc15ca2016 100644 --- a/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery +++ b/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery @@ -39,11 +39,28 @@ Binding 0: "index" ] }, + "a_bool": { + "type": "boolean" + }, + "a_date": { + "type": "string", + "format": "date-time" + }, + "a_real": { + "type": "number" + }, + "a_ts": { + "type": "string", + "format": "date-time" + }, + "a_tstz": { + "type": "string", + "format": "date-time" + }, "id": { "type": "integer" } - }, - "x-infer-schema": true + } }, "key": [ "/id" diff --git a/source-redshift-batch/.snapshots/TestKeyDiscovery b/source-redshift-batch/.snapshots/TestKeyDiscovery index 66d5f46c61..88ff67496a 100644 --- a/source-redshift-batch/.snapshots/TestKeyDiscovery +++ b/source-redshift-batch/.snapshots/TestKeyDiscovery @@ -43,6 +43,9 @@ Binding 0: "index" ] }, + "data": { + "type": "string" + }, "k_bigint": { "type": "integer" }, @@ -58,8 +61,7 @@ Binding 0: "k_str": { "type": "string" } - }, - "x-infer-schema": true + } }, "key": [ "/k_smallint", diff --git a/source-redshift-batch/.snapshots/TestKeylessDiscovery b/source-redshift-batch/.snapshots/TestKeylessDiscovery new file mode 100644 index 0000000000..53a6bb0a60 --- /dev/null +++ b/source-redshift-batch/.snapshots/TestKeylessDiscovery @@ -0,0 +1,73 @@ +Binding 0: +{ + "resource_config_json": { + "name": "test_keyless_discovery_10352", + "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"keyless_discovery_10352\"\n {{- else -}}\n SELECT * FROM \"test\".\"keyless_discovery_10352\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"keyless_discovery_10352\";\n{{- end}}\n", + "cursor": null + }, + "resource_path": [ + "test_keyless_discovery_10352" + ], + "collection": { + "name": "acmeCo/test/test_keyless_discovery_10352", + "read_schema_json": { + "type": "object", + "required": [ + "_meta" + ], + "properties": { + "_meta": { + "$schema": "http://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/estuary/connectors/source-redshift-batch/document-metadata", + "properties": { + "polled": { + "type": "string", + "format": "date-time", + "title": "Polled Timestamp", + "description": "The time at which the update query which produced this document as executed." + }, + "index": { + "type": "integer", + "title": "Result Index", + "description": "The index of this document within the query execution which produced it." + } + }, + "type": "object", + "required": [ + "polled", + "index" + ] + }, + "v_bigint": { + "type": "integer" + }, + "v_bool": { + "type": "boolean" + }, + "v_int": { + "type": "integer" + }, + "v_smallint": { + "type": "integer" + }, + "v_str": { + "type": "string" + }, + "v_text": { + "type": "string" + }, + "v_ts": { + "type": "string", + "format": "date-time" + }, + "v_tstz": { + "type": "string", + "format": "date-time" + } + } + }, + "projections": null + }, + "state_key": "test_keyless_discovery_10352" + } + diff --git a/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn b/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn index fa1c58d051..471460386e 100644 --- a/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn +++ b/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn @@ -39,11 +39,13 @@ Binding 0: "index" ] }, + "data": { + "type": "string" + }, "id": { "type": "integer" } - }, - "x-infer-schema": true + } }, "key": [ "/id" diff --git a/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered b/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered index fa1c58d051..471460386e 100644 --- a/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered +++ b/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered @@ -39,11 +39,13 @@ Binding 0: "index" ] }, + "data": { + "type": "string" + }, "id": { "type": "integer" } - }, - "x-infer-schema": true + } }, "key": [ "/id" diff --git a/source-redshift-batch/driver.go b/source-redshift-batch/driver.go index 1ae403d7e9..ecc80ea6d3 100644 --- a/source-redshift-batch/driver.go +++ b/source-redshift-batch/driver.go @@ -86,10 +86,7 @@ type documentMetadata struct { Index int `json:"index" jsonschema:"title=Result Index,description=The index of this document within the query execution which produced it."` } -// The fallback collection key just refers to the polling iteration and result index of each document. -var fallbackKey = []string{"/_meta/polled", "/_meta/index"} - -func generateCollectionSchema(keyColumns []string, columnTypes map[string]*jsonschema.Schema) (json.RawMessage, error) { +func generateCollectionSchema(keyColumns []string, columnTypes map[string]columnType) (json.RawMessage, error) { // Generate schema for the metadata via reflection var reflector = jsonschema.Reflector{ ExpandedStruct: true, @@ -99,18 +96,12 @@ func generateCollectionSchema(keyColumns []string, columnTypes map[string]*jsons metadataSchema.Definitions = nil metadataSchema.AdditionalProperties = nil - var required = []string{"_meta"} + var required = append([]string{"_meta"}, keyColumns...) var properties = map[string]*jsonschema.Schema{ "_meta": metadataSchema, } - - for _, colName := range keyColumns { - var columnType = columnTypes[colName] - if columnType == nil { - return nil, fmt.Errorf("unable to add key column %q to schema: type unknown", colName) - } - properties[colName] = columnType - required = append(required, colName) + for colName, colType := range columnTypes { + properties[colName] = colType.JSONSchema() } var schema = &jsonschema.Schema{ @@ -118,8 +109,7 @@ func generateCollectionSchema(keyColumns []string, columnTypes map[string]*jsons Required: required, AdditionalProperties: nil, Extras: map[string]interface{}{ - "properties": properties, - "x-infer-schema": true, + "properties": properties, }, } @@ -131,14 +121,6 @@ func generateCollectionSchema(keyColumns []string, columnTypes map[string]*jsons return json.RawMessage(bs), nil } -var minimalSchema = func() json.RawMessage { - var schema, err = generateCollectionSchema(nil, nil) - if err != nil { - panic(err) - } - return schema -}() - // Spec returns metadata about the capture connector. func (drv *BatchSQLDriver) Spec(ctx context.Context, req *pc.Request_Spec) (*pc.Response_Spec, error) { resourceSchema, err := schemagen.GenerateSchema("Batch SQL Resource Spec", &Resource{}).MarshalJSON() @@ -174,24 +156,41 @@ func (drv *BatchSQLDriver) Discover(ctx context.Context, req *pc.Request_Discove } defer db.Close() + // Run discovery queries tables, err := discoverTables(ctx, db, cfg.Advanced.DiscoverSchemas) if err != nil { return nil, fmt.Errorf("error listing tables: %w", err) } + columns, err := discoverColumns(ctx, db, cfg.Advanced.DiscoverSchemas) + if err != nil { + return nil, fmt.Errorf("error listing columns: %w", err) + } keys, err := discoverPrimaryKeys(ctx, db, cfg.Advanced.DiscoverSchemas) if err != nil { return nil, fmt.Errorf("error listing primary keys: %w", err) } - // We rebuild this map even though `discoverPrimaryKeys` already built and - // discarded it, in order to keep the `discoverTables` / `discoverPrimaryKeys` - // interface as stupidly simple as possible, which ought to make it just that - // little bit easier to do this generically for multiple databases. - var keysByTable = make(map[string]*discoveredPrimaryKey) + // Aggregate column information by table + var columnsByTable = make(map[string][]*discoveredColumn) + for _, column := range columns { + var tableID = column.Schema + "." + column.Table + columnsByTable[tableID] = append(columnsByTable[tableID], column) + if column.Index != len(columnsByTable[tableID]) { + return nil, fmt.Errorf("internal error: column %q of table %q appears out of order", column.Name, tableID) + } + } + + // Aggregate primary-key information by table + var keysByTable = make(map[string][]*discoveredPrimaryKey) for _, key := range keys { - keysByTable[key.Schema+"."+key.Table] = key + var tableID = key.Schema + "." + key.Table + keysByTable[tableID] = append(keysByTable[tableID], key) + if key.Index != len(keysByTable[tableID]) { + return nil, fmt.Errorf("internal error: primary key column %q of table %q appears out of order", key.Column, tableID) + } } + // Generate discovery resource and collection schema for this table var bindings []*pc.Response_Discovered_Binding for _, table := range tables { var tableID = table.Schema + "." + table.Name @@ -211,26 +210,32 @@ func (drv *BatchSQLDriver) Discover(ctx context.Context, req *pc.Request_Discove return nil, fmt.Errorf("error serializing resource spec: %w", err) } - // Try to generate a useful collection schema, but on error fall back to the - // minimal schema with the default key [/_meta/polled, /_meta/index]. - var collectionSchema = minimalSchema - var collectionKey = fallbackKey - if tableKey, ok := keysByTable[tableID]; ok { - if generatedSchema, err := generateCollectionSchema(tableKey.Columns, tableKey.ColumnTypes); err == nil { - collectionSchema = generatedSchema - collectionKey = nil - for _, colName := range tableKey.Columns { - collectionKey = append(collectionKey, primaryKeyToCollectionKey(colName)) - } - } else { - log.WithFields(log.Fields{"table": tableID, "err": err}).Warn("unable to generate collection schema") - } + // Generate a collection schema from the column types and key column names of this table. + var keyColumns []string + for _, key := range keysByTable[tableID] { + keyColumns = append(keyColumns, key.Column) + } + + var columnTypes = make(map[string]columnType) + for _, column := range columnsByTable[tableID] { + columnTypes[column.Name] = column.DataType + } + + generatedSchema, err := generateCollectionSchema(keyColumns, columnTypes) + if err != nil { + log.WithFields(log.Fields{"table": tableID, "err": err}).Warn("unable to generate collection schema") + continue + } + + var collectionKey []string + for _, colName := range keyColumns { + collectionKey = append(collectionKey, primaryKeyToCollectionKey(colName)) } bindings = append(bindings, &pc.Response_Discovered_Binding{ RecommendedName: recommendedName, ResourceConfigJson: resourceConfigJSON, - DocumentSchemaJson: collectionSchema, + DocumentSchemaJson: generatedSchema, Key: collectionKey, ResourcePath: []string{res.Name}, }) @@ -259,7 +264,7 @@ func discoverTables(ctx context.Context, db *sql.DB, discoverSchemas []string) ( var query = new(strings.Builder) var args []any - fmt.Fprintf(query, "SELECT n.nspname AS table_schema,") + fmt.Fprintf(query, "SELECT nc.nspname AS table_schema,") fmt.Fprintf(query, " c.relname AS table_name,") fmt.Fprintf(query, " CASE") fmt.Fprintf(query, " WHEN c.relkind = ANY (ARRAY['r'::\"char\", 'p'::\"char\"]) THEN 'BASE TABLE'::text") @@ -268,12 +273,13 @@ func discoverTables(ctx context.Context, db *sql.DB, discoverSchemas []string) ( fmt.Fprintf(query, " ELSE ''::text") fmt.Fprintf(query, " END::information_schema.character_data AS table_type") fmt.Fprintf(query, " FROM pg_catalog.pg_class c") - fmt.Fprintf(query, " JOIN pg_catalog.pg_namespace n ON (n.oid = c.relnamespace)") - fmt.Fprintf(query, " WHERE n.nspname NOT IN ('pg_catalog', 'pg_internal', 'information_schema', 'catalog_history', 'cron')") - fmt.Fprintf(query, " AND c.relkind IN ('r', 'p', 'v', 'f')") + fmt.Fprintf(query, " JOIN pg_catalog.pg_namespace nc ON (nc.oid = c.relnamespace)") + fmt.Fprintf(query, " WHERE c.relkind IN ('r', 'p', 'v', 'f')") if len(discoverSchemas) > 0 { - fmt.Fprintf(query, " AND n.nspname = ANY ($1)") + fmt.Fprintf(query, " AND nc.nspname = ANY ($1)") args = append(args, discoverSchemas) + } else { + fmt.Fprintf(query, " AND nc.nspname NOT IN ('pg_catalog', 'pg_internal', 'information_schema', 'catalog_history', 'cron')") } fmt.Fprintf(query, ";") @@ -298,11 +304,122 @@ func discoverTables(ctx context.Context, db *sql.DB, discoverSchemas []string) ( return tables, nil } +type discoveredColumn struct { + Schema string // The schema in which the table resides + Table string // The name of the table with this column + Name string // The name of the column + Index int // The ordinal position of the column within a row + IsNullable bool // Whether the column can be null + DataType columnType // The datatype of the column + Description *string // The description of the column, if present and known +} + +type columnType interface { + JSONSchema() *jsonschema.Schema +} + +type basicColumnType struct { + jsonType string + contentEncoding string + format string + nullable bool + description string +} + +func (ct *basicColumnType) JSONSchema() *jsonschema.Schema { + var sch = &jsonschema.Schema{ + Format: ct.format, + Extras: make(map[string]interface{}), + } + + if ct.contentEncoding != "" { + sch.Extras["contentEncoding"] = ct.contentEncoding // New in 2019-09. + } + + if ct.jsonType == "" { + // No type constraint. + } else if !ct.nullable { + sch.Type = ct.jsonType + } else { + sch.Extras["type"] = []string{ct.jsonType, "null"} + } + return sch +} + +func discoverColumns(ctx context.Context, db *sql.DB, discoverSchemas []string) ([]*discoveredColumn, error) { + var query = new(strings.Builder) + var args []any + fmt.Fprintf(query, "SELECT nc.nspname as table_schema,") + fmt.Fprintf(query, " c.relname as table_name,") + fmt.Fprintf(query, " a.attname as column_name,") + fmt.Fprintf(query, " a.attnum as column_index,") + fmt.Fprintf(query, " NOT (a.attnotnull OR (t.typtype = 'd' AND t.typnotnull)) AS is_nullable,") + fmt.Fprintf(query, " COALESCE(bt.typname, t.typname) AS udt_name,") + fmt.Fprintf(query, " t.typtype::text AS typtype") + fmt.Fprintf(query, " FROM pg_catalog.pg_attribute a") + fmt.Fprintf(query, " JOIN pg_catalog.pg_type t ON a.atttypid = t.oid") + fmt.Fprintf(query, " JOIN pg_catalog.pg_class c ON a.attrelid = c.oid") + fmt.Fprintf(query, " JOIN pg_catalog.pg_namespace nc ON c.relnamespace = nc.oid") + fmt.Fprintf(query, " LEFT JOIN (pg_catalog.pg_type bt JOIN pg_namespace nbt ON bt.typnamespace = nbt.oid)") + fmt.Fprintf(query, " ON t.typtype = 'd'::\"char\" AND t.typbasetype = bt.oid") + fmt.Fprintf(query, " WHERE a.attnum > 0") + fmt.Fprintf(query, " AND NOT a.attisdropped") + fmt.Fprintf(query, " AND c.relkind IN ('r', 'p', 'v', 'f')") + fmt.Fprintf(query, " AND nc.nspname NOT IN ('pg_catalog', 'pg_internal', 'information_schema', 'catalog_history', 'cron')") + if len(discoverSchemas) > 0 { + fmt.Fprintf(query, " AND nc.nspname = ANY ($1)") + args = append(args, discoverSchemas) + } + fmt.Fprintf(query, " ORDER BY nc.nspname, c.relname, a.attnum;") + + rows, err := db.QueryContext(ctx, query.String(), args...) + if err != nil { + return nil, fmt.Errorf("error executing discovery query %q: %w", query.String(), err) + } + defer rows.Close() + + var columns []*discoveredColumn + for rows.Next() { + var tableSchema, tableName, columnName string + var columnIndex int + var isNullable bool + var typeName, typeType string + if err := rows.Scan(&tableSchema, &tableName, &columnName, &columnIndex, &isNullable, &typeName, &typeType); err != nil { + return nil, fmt.Errorf("error scanning result row: %w", err) + } + + // Decode column type information into a usable form + var dataType columnType + switch typeType { + case "e": // enum values are captured as strings + dataType = &basicColumnType{jsonType: "string"} + case "r", "m": // ranges and multiranges are captured as strings + dataType = &basicColumnType{jsonType: "string"} + default: + var ok bool + dataType, ok = databaseTypeToJSON[typeName] + if !ok { + dataType = &basicColumnType{description: fmt.Sprintf("using catch-all schema for unknown type %q", typeName)} + } + } + + columns = append(columns, &discoveredColumn{ + Schema: tableSchema, + Table: tableName, + Name: columnName, + Index: columnIndex, + IsNullable: isNullable, + DataType: dataType, + }) + } + return columns, nil +} + type discoveredPrimaryKey struct { - Schema string - Table string - Columns []string - ColumnTypes map[string]*jsonschema.Schema + Schema string + Table string + Column string + Index int } func discoverPrimaryKeys(ctx context.Context, db *sql.DB, discoverSchemas []string) ([]*discoveredPrimaryKey, error) { @@ -312,22 +429,15 @@ func discoverPrimaryKeys(ctx context.Context, db *sql.DB, discoverSchemas []stri fmt.Fprintf(query, "SELECT nr.nspname::information_schema.sql_identifier AS table_schema,") fmt.Fprintf(query, " r.relname::information_schema.sql_identifier AS table_name,") fmt.Fprintf(query, " a.attname::information_schema.sql_identifier AS column_name,") - fmt.Fprintf(query, " pos.n::information_schema.cardinal_number AS ordinal_position,") - // The handling of type names here is a bit weak, refer to how the `information_schema.columns` - // view computes its `data_type` column for a more comprehensive approach. But in practice we - // don't need all of that complexity just to identify basic integer columns, so this ought to - // be sufficient for now. - fmt.Fprintf(query, " t.typname AS type_name") + fmt.Fprintf(query, " pos.n::information_schema.cardinal_number AS ordinal_position") fmt.Fprintf(query, " FROM pg_namespace nr,") fmt.Fprintf(query, " pg_class r,") fmt.Fprintf(query, " pg_attribute a,") - fmt.Fprintf(query, " pg_type t,") fmt.Fprintf(query, " pg_constraint c,") fmt.Fprintf(query, " generate_series(1,100,1) pos(n)") fmt.Fprintf(query, " WHERE nr.oid = r.relnamespace") fmt.Fprintf(query, " AND r.oid = a.attrelid") fmt.Fprintf(query, " AND r.oid = c.conrelid") - fmt.Fprintf(query, " AND t.oid = a.atttypid") fmt.Fprintf(query, " AND c.conkey[pos.n] = a.attnum") fmt.Fprintf(query, " AND NOT a.attisdropped") fmt.Fprintf(query, " AND c.contype = 'p'::\"char\"") @@ -344,50 +454,69 @@ func discoverPrimaryKeys(ctx context.Context, db *sql.DB, discoverSchemas []stri } defer rows.Close() - var keysByTable = make(map[string]*discoveredPrimaryKey) + var keys []*discoveredPrimaryKey for rows.Next() { - var tableSchema, tableName, columnName, dataType string + var tableSchema, tableName, columnName string var ordinalPosition int - if err := rows.Scan(&tableSchema, &tableName, &columnName, &ordinalPosition, &dataType); err != nil { + if err := rows.Scan(&tableSchema, &tableName, &columnName, &ordinalPosition); err != nil { return nil, fmt.Errorf("error scanning result row: %w", err) } - var tableID = tableSchema + "." + tableName - var keyInfo = keysByTable[tableID] - if keyInfo == nil { - keyInfo = &discoveredPrimaryKey{ - Schema: tableSchema, - Table: tableName, - ColumnTypes: make(map[string]*jsonschema.Schema), - } - keysByTable[tableID] = keyInfo - } - keyInfo.Columns = append(keyInfo.Columns, columnName) - if jsonSchema, ok := databaseTypeToJSON[dataType]; ok { - keyInfo.ColumnTypes[columnName] = jsonSchema - } else { - // Assume unknown types are strings, because it's almost always a string. - // (Or it's an object, but those can't be Flow collection keys anyway) - keyInfo.ColumnTypes[columnName] = &jsonschema.Schema{Type: "string"} - } - if ordinalPosition != len(keyInfo.Columns) { - return nil, fmt.Errorf("primary key column %q (of table %q) appears out of order", columnName, tableID) - } - } - - var keys []*discoveredPrimaryKey - for _, key := range keysByTable { - keys = append(keys, key) + keys = append(keys, &discoveredPrimaryKey{ + Schema: tableSchema, + Table: tableName, + Column: columnName, + Index: ordinalPosition, + }) } return keys, nil } -var databaseTypeToJSON = map[string]*jsonschema.Schema{ - "int2": {Type: "integer"}, - "int4": {Type: "integer"}, - "int8": {Type: "integer"}, - "bool": {Type: "boolean"}, - "varchar": {Type: "string"}, +var databaseTypeToJSON = map[string]columnType{ + "bool": &basicColumnType{jsonType: "boolean"}, + + "int2": &basicColumnType{jsonType: "integer"}, + "int4": &basicColumnType{jsonType: "integer"}, + "int8": &basicColumnType{jsonType: "integer"}, + + "numeric": &basicColumnType{jsonType: "string", format: "number"}, + "float4": &basicColumnType{jsonType: "number"}, + "float8": &basicColumnType{jsonType: "number"}, + + "varchar": &basicColumnType{jsonType: "string"}, + "bpchar": &basicColumnType{jsonType: "string"}, + "text": &basicColumnType{jsonType: "string"}, + "bytea": &basicColumnType{jsonType: "string", contentEncoding: "base64"}, + "xml": &basicColumnType{jsonType: "string"}, + "bit": &basicColumnType{jsonType: "string"}, + "varbit": &basicColumnType{jsonType: "string"}, + + "json": &basicColumnType{}, + "jsonb": &basicColumnType{}, + "jsonpath": &basicColumnType{jsonType: "string"}, + + // Domain-Specific Types + "date": &basicColumnType{jsonType: "string", format: "date-time"}, + "timestamp": &basicColumnType{jsonType: "string", format: "date-time"}, + "timestamptz": &basicColumnType{jsonType: "string", format: "date-time"}, + "time": &basicColumnType{jsonType: "integer"}, + "timetz": &basicColumnType{jsonType: "string", format: "time"}, + "interval": &basicColumnType{jsonType: "string"}, + "money": &basicColumnType{jsonType: "string"}, + "point": &basicColumnType{jsonType: "string"}, + "line": &basicColumnType{jsonType: "string"}, + "lseg": &basicColumnType{jsonType: "string"}, + "box": &basicColumnType{jsonType: "string"}, + "path": &basicColumnType{jsonType: "string"}, + "polygon": &basicColumnType{jsonType: "string"}, + "circle": &basicColumnType{jsonType: "string"}, + "inet": &basicColumnType{jsonType: "string"}, + "cidr": &basicColumnType{jsonType: "string"}, + "macaddr": &basicColumnType{jsonType: "string"}, + "macaddr8": &basicColumnType{jsonType: "string"}, + "tsvector": &basicColumnType{jsonType: "string"}, + "tsquery": &basicColumnType{jsonType: "string"}, + "uuid": &basicColumnType{jsonType: "string", format: "uuid"}, } var catalogNameSanitizerRe = regexp.MustCompile(`(?i)[^a-z0-9\-_.]`) diff --git a/source-redshift-batch/main_test.go b/source-redshift-batch/main_test.go index c2c66396e0..5bbfb56325 100644 --- a/source-redshift-batch/main_test.go +++ b/source-redshift-batch/main_test.go @@ -194,6 +194,20 @@ func TestKeyDiscovery(t *testing.T) { snapshotBindings(t, discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID))) } +func TestKeylessDiscovery(t *testing.T) { + var ctx, cs = context.Background(), testCaptureSpec(t) + var control = testControlClient(ctx, t) + var uniqueID = "10352" + var tableName = fmt.Sprintf("test.keyless_discovery_%s", uniqueID) + + executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) + t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) + executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(v_smallint SMALLINT, v_int INTEGER, v_bigint BIGINT, v_bool BOOLEAN, v_str VARCHAR(8), v_ts TIMESTAMP, v_tstz TIMESTAMP WITH TIME ZONE, v_text TEXT)", tableName)) + + cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{"test"} + snapshotBindings(t, discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID))) +} + func testControlClient(ctx context.Context, t testing.TB) *sql.DB { t.Helper() if os.Getenv("TEST_DATABASE") != "yes" { @@ -208,7 +222,7 @@ func testControlClient(ctx context.Context, t testing.TB) *sql.DB { if controlPass == "" { controlPass = *dbCapturePass } - var controlURI = fmt.Sprintf(`postgres://%s:%s@%s/%s?sslmode=require`, controlUser, controlPass, *dbAddress, *dbName) + var controlURI = fmt.Sprintf(`postgres://%s:%s@%s/%s`, controlUser, controlPass, *dbAddress, *dbName) log.WithField("uri", controlURI).Debug("opening database control connection") var conn, err = sql.Open("pgx", controlURI) require.NoError(t, err) From 22fb95794f1bc46f52b60546e46be63962867805 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Thu, 21 Mar 2024 15:41:02 -0500 Subject: [PATCH 38/96] source-redshift-batch: Be more consistent about `nspname` filter The Redshift discovery queries for tables, columns, and primary keys all have a similar "if schema discovery whitelisting is in use ..." branch to their query generation logic, but with subtle differences. There is no reason these need to be substantially different, so this just makes the logic the same in all three cases. --- source-redshift-batch/driver.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source-redshift-batch/driver.go b/source-redshift-batch/driver.go index ecc80ea6d3..e6c7a31599 100644 --- a/source-redshift-batch/driver.go +++ b/source-redshift-batch/driver.go @@ -365,10 +365,11 @@ func discoverColumns(ctx context.Context, db *sql.DB, discoverSchemas []string) fmt.Fprintf(query, " WHERE a.attnum > 0") fmt.Fprintf(query, " AND NOT a.attisdropped") fmt.Fprintf(query, " AND c.relkind IN ('r', 'p', 'v', 'f')") - fmt.Fprintf(query, " AND nc.nspname NOT IN ('pg_catalog', 'pg_internal', 'information_schema', 'catalog_history', 'cron')") if len(discoverSchemas) > 0 { fmt.Fprintf(query, " AND nc.nspname = ANY ($1)") args = append(args, discoverSchemas) + } else { + fmt.Fprintf(query, " AND nc.nspname NOT IN ('pg_catalog', 'pg_internal', 'information_schema', 'catalog_history', 'cron')") } fmt.Fprintf(query, " ORDER BY nc.nspname, c.relname, a.attnum;") @@ -445,6 +446,8 @@ func discoverPrimaryKeys(ctx context.Context, db *sql.DB, discoverSchemas []stri if len(discoverSchemas) > 0 { fmt.Fprintf(query, " AND nr.nspname = ANY ($1)") args = append(args, discoverSchemas) + } else { + fmt.Fprintf(query, " AND nr.nspname NOT IN ('pg_catalog', 'pg_internal', 'information_schema', 'catalog_history', 'cron')") } fmt.Fprintf(query, " ORDER BY r.relname, pos.n;") From 1cd3c4aa60904d6b0c79a3ce051cc0e98bbaf030 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Thu, 21 Mar 2024 16:29:46 -0500 Subject: [PATCH 39/96] source-redshift-batch: Omit empty cursors in discovered resources The cursor values will always be empty, and we want that to be represented by omitting the property from the discovered resource spec entirely rather than with an explicit null. Also makes the same change to `source-postgres-batch`. It doesn't matter there because discovered resources always have the default cursor `["txid"]` but the fact that I left off the `omitempty` in that case is why the Redshift connector got this wrong. --- source-postgres-batch/.snapshots/TestSpec | 3 +-- source-postgres-batch/driver.go | 2 +- source-redshift-batch/.snapshots/TestBasicCapture-Discovery | 3 +-- source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery | 3 +-- source-redshift-batch/.snapshots/TestKeyDiscovery | 3 +-- source-redshift-batch/.snapshots/TestKeylessDiscovery | 3 +-- source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn | 3 +-- source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered | 3 +-- source-redshift-batch/.snapshots/TestSpec | 3 +-- source-redshift-batch/driver.go | 2 +- 10 files changed, 10 insertions(+), 18 deletions(-) diff --git a/source-postgres-batch/.snapshots/TestSpec b/source-postgres-batch/.snapshots/TestSpec index 2a66de90fd..3eb914714e 100644 --- a/source-postgres-batch/.snapshots/TestSpec +++ b/source-postgres-batch/.snapshots/TestSpec @@ -109,8 +109,7 @@ "type": "object", "required": [ "name", - "template", - "cursor" + "template" ], "title": "Batch SQL Resource Spec" }, diff --git a/source-postgres-batch/driver.go b/source-postgres-batch/driver.go index 403230d226..ca32ba9177 100644 --- a/source-postgres-batch/driver.go +++ b/source-postgres-batch/driver.go @@ -47,7 +47,7 @@ type BatchSQLDriver struct { type Resource struct { Name string `json:"name" jsonschema:"title=Name,description=The unique name of this resource." jsonschema_extras:"order=0"` Template string `json:"template" jsonschema:"title=Query Template,description=The query template (pkg.go.dev/text/template) which will be rendered and then executed." jsonschema_extras:"multiline=true,order=3"` - Cursor []string `json:"cursor" jsonschema:"title=Cursor Columns,description=The names of columns which should be persisted between query executions as a cursor." jsonschema_extras:"order=2"` + Cursor []string `json:"cursor,omitempty" jsonschema:"title=Cursor Columns,description=The names of columns which should be persisted between query executions as a cursor." jsonschema_extras:"order=2"` PollSchedule string `json:"poll,omitempty" jsonschema:"title=Polling Schedule,description=When and how often to execute the fetch query (overrides the connector default setting). Accepts a Go duration string like '5m' or '6h' for frequency-based polling or a string like 'daily at 12:34Z' to poll at a specific time (specified in UTC) every day." jsonschema_extras:"order=1,pattern=^([-+]?([0-9]+([.][0-9]+)?(h|m|s|ms))+|daily at [0-9][0-9]?:[0-9]{2}Z)$"` } diff --git a/source-redshift-batch/.snapshots/TestBasicCapture-Discovery b/source-redshift-batch/.snapshots/TestBasicCapture-Discovery index ac8bbc7f1f..43e55f91b7 100644 --- a/source-redshift-batch/.snapshots/TestBasicCapture-Discovery +++ b/source-redshift-batch/.snapshots/TestBasicCapture-Discovery @@ -2,8 +2,7 @@ Binding 0: { "resource_config_json": { "name": "test_basic_capture_826935", - "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"basic_capture_826935\"\n {{- else -}}\n SELECT * FROM \"test\".\"basic_capture_826935\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"basic_capture_826935\";\n{{- end}}\n", - "cursor": null + "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"basic_capture_826935\"\n {{- else -}}\n SELECT * FROM \"test\".\"basic_capture_826935\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"basic_capture_826935\";\n{{- end}}\n" }, "resource_path": [ "test_basic_capture_826935" diff --git a/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery b/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery index cc15ca2016..3fcb2b5f61 100644 --- a/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery +++ b/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery @@ -2,8 +2,7 @@ Binding 0: { "resource_config_json": { "name": "test_basic_datatypes_13111208", - "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"basic_datatypes_13111208\"\n {{- else -}}\n SELECT * FROM \"test\".\"basic_datatypes_13111208\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"basic_datatypes_13111208\";\n{{- end}}\n", - "cursor": null + "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"basic_datatypes_13111208\"\n {{- else -}}\n SELECT * FROM \"test\".\"basic_datatypes_13111208\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"basic_datatypes_13111208\";\n{{- end}}\n" }, "resource_path": [ "test_basic_datatypes_13111208" diff --git a/source-redshift-batch/.snapshots/TestKeyDiscovery b/source-redshift-batch/.snapshots/TestKeyDiscovery index 88ff67496a..08f4957a24 100644 --- a/source-redshift-batch/.snapshots/TestKeyDiscovery +++ b/source-redshift-batch/.snapshots/TestKeyDiscovery @@ -2,8 +2,7 @@ Binding 0: { "resource_config_json": { "name": "test_key_discovery_329932", - "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"key_discovery_329932\"\n {{- else -}}\n SELECT * FROM \"test\".\"key_discovery_329932\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"key_discovery_329932\";\n{{- end}}\n", - "cursor": null + "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"key_discovery_329932\"\n {{- else -}}\n SELECT * FROM \"test\".\"key_discovery_329932\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"key_discovery_329932\";\n{{- end}}\n" }, "resource_path": [ "test_key_discovery_329932" diff --git a/source-redshift-batch/.snapshots/TestKeylessDiscovery b/source-redshift-batch/.snapshots/TestKeylessDiscovery index 53a6bb0a60..59e20c095a 100644 --- a/source-redshift-batch/.snapshots/TestKeylessDiscovery +++ b/source-redshift-batch/.snapshots/TestKeylessDiscovery @@ -2,8 +2,7 @@ Binding 0: { "resource_config_json": { "name": "test_keyless_discovery_10352", - "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"keyless_discovery_10352\"\n {{- else -}}\n SELECT * FROM \"test\".\"keyless_discovery_10352\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"keyless_discovery_10352\";\n{{- end}}\n", - "cursor": null + "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"keyless_discovery_10352\"\n {{- else -}}\n SELECT * FROM \"test\".\"keyless_discovery_10352\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"keyless_discovery_10352\";\n{{- end}}\n" }, "resource_path": [ "test_keyless_discovery_10352" diff --git a/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn b/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn index 471460386e..cbb7ec02ac 100644 --- a/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn +++ b/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn @@ -2,8 +2,7 @@ Binding 0: { "resource_config_json": { "name": "test_schema_filtering_22492", - "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\"\n {{- else -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\";\n{{- end}}\n", - "cursor": null + "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\"\n {{- else -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\";\n{{- end}}\n" }, "resource_path": [ "test_schema_filtering_22492" diff --git a/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered b/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered index 471460386e..cbb7ec02ac 100644 --- a/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered +++ b/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered @@ -2,8 +2,7 @@ Binding 0: { "resource_config_json": { "name": "test_schema_filtering_22492", - "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\"\n {{- else -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\";\n{{- end}}\n", - "cursor": null + "template": "{{if .CursorFields -}}\n {{- if .IsFirstQuery -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\"\n {{- else -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\"\n\t{{- range $i, $k := $.CursorFields -}}\n\t {{- if eq $i 0}} WHERE ({{else}}) OR ({{end -}}\n {{- range $j, $n := $.CursorFields -}}\n\t\t{{- if lt $j $i -}}\n\t\t {{$n}} = ${{add $j 1}} AND {{end -}}\n\t {{- end -}}\n\t {{$k}} \u003e ${{add $i 1}}\n\t{{- end -}}\n\t) \n {{- end}} ORDER BY {{range $i, $k := $.CursorFields}}{{if gt $i 0}}, {{end}}{{$k}}{{end -}};\n{{- else -}}\n SELECT * FROM \"test\".\"schema_filtering_22492\";\n{{- end}}\n" }, "resource_path": [ "test_schema_filtering_22492" diff --git a/source-redshift-batch/.snapshots/TestSpec b/source-redshift-batch/.snapshots/TestSpec index 79587c3470..e6d4343aed 100644 --- a/source-redshift-batch/.snapshots/TestSpec +++ b/source-redshift-batch/.snapshots/TestSpec @@ -109,8 +109,7 @@ "type": "object", "required": [ "name", - "template", - "cursor" + "template" ], "title": "Batch SQL Resource Spec" }, diff --git a/source-redshift-batch/driver.go b/source-redshift-batch/driver.go index e6c7a31599..0af5b6afe3 100644 --- a/source-redshift-batch/driver.go +++ b/source-redshift-batch/driver.go @@ -47,7 +47,7 @@ type BatchSQLDriver struct { type Resource struct { Name string `json:"name" jsonschema:"title=Name,description=The unique name of this resource." jsonschema_extras:"order=0"` Template string `json:"template" jsonschema:"title=Query Template,description=The query template (pkg.go.dev/text/template) which will be rendered and then executed." jsonschema_extras:"multiline=true,order=3"` - Cursor []string `json:"cursor" jsonschema:"title=Cursor Columns,description=The names of columns which should be persisted between query executions as a cursor." jsonschema_extras:"order=2"` + Cursor []string `json:"cursor,omitempty" jsonschema:"title=Cursor Columns,description=The names of columns which should be persisted between query executions as a cursor." jsonschema_extras:"order=2"` PollSchedule string `json:"poll,omitempty" jsonschema:"title=Polling Schedule,description=When and how often to execute the fetch query (overrides the connector default setting). Accepts a Go duration string like '5m' or '6h' for frequency-based polling or a string like 'daily at 12:34Z' to poll at a specific time (specified in UTC) every day." jsonschema_extras:"order=1,pattern=^([-+]?([0-9]+([.][0-9]+)?(h|m|s|ms))+|daily at [0-9][0-9]?:[0-9]{2}Z)$"` } From f4b7ad2150323f8b046a30ce86778b78db2b9232 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Thu, 21 Mar 2024 17:54:44 -0500 Subject: [PATCH 40/96] source-redshift-batch: Run discovery queries in parallel The parallel execution is done using a fairly clunky and verbose pattern (running each discovery query in a separate `errgroup` worker thread, using a single-element buffered channel for each thread's result) because it was what I could confidently bang out in 15 minutes. We should consider replacing the whole mess with a suitable 'promise' abstraction helper package if one exists. --- source-redshift-batch/driver.go | 48 ++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/source-redshift-batch/driver.go b/source-redshift-batch/driver.go index 0af5b6afe3..56e0eb44e2 100644 --- a/source-redshift-batch/driver.go +++ b/source-redshift-batch/driver.go @@ -156,19 +156,43 @@ func (drv *BatchSQLDriver) Discover(ctx context.Context, req *pc.Request_Discove } defer db.Close() - // Run discovery queries - tables, err := discoverTables(ctx, db, cfg.Advanced.DiscoverSchemas) - if err != nil { - return nil, fmt.Errorf("error listing tables: %w", err) - } - columns, err := discoverColumns(ctx, db, cfg.Advanced.DiscoverSchemas) - if err != nil { - return nil, fmt.Errorf("error listing columns: %w", err) - } - keys, err := discoverPrimaryKeys(ctx, db, cfg.Advanced.DiscoverSchemas) - if err != nil { - return nil, fmt.Errorf("error listing primary keys: %w", err) + // Run discovery queries in parallel for lower discovery latency on large databases. + // TODO(wgd): See if there's a nice context-and-errors-aware promise library we could use + // here instead of just doing the same copy-paste channels-and-errgroup pattern thrice. + var tablesCh = make(chan []*discoveredTable, 1) + var columnsCh = make(chan []*discoveredColumn, 1) + var keysCh = make(chan []*discoveredPrimaryKey, 1) + var workerGroup, workerCtx = errgroup.WithContext(ctx) + workerGroup.Go(func() error { + tables, err := discoverTables(workerCtx, db, cfg.Advanced.DiscoverSchemas) + if err != nil { + return fmt.Errorf("error listing tables: %w", err) + } + tablesCh <- tables + return nil + }) + workerGroup.Go(func() error { + columns, err := discoverColumns(workerCtx, db, cfg.Advanced.DiscoverSchemas) + if err != nil { + return fmt.Errorf("error listing columns: %w", err) + } + columnsCh <- columns + return nil + }) + workerGroup.Go(func() error { + keys, err := discoverPrimaryKeys(workerCtx, db, cfg.Advanced.DiscoverSchemas) + if err != nil { + return fmt.Errorf("error listing primary keys: %w", err) + } + keysCh <- keys + return nil + }) + if err := workerGroup.Wait(); err != nil { + return nil, err } + var tables = <-tablesCh + var columns = <-columnsCh + var keys = <-keysCh // Aggregate column information by table var columnsByTable = make(map[string][]*discoveredColumn) From 6470e9d7bc1c197fdca8609c7217a900f1caf61e Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 22 Mar 2024 11:07:44 +0000 Subject: [PATCH 41/96] materialize-sql: close client in Validate --- materialize-sql/driver.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/materialize-sql/driver.go b/materialize-sql/driver.go index dba27eb9ec..ab43140cd1 100644 --- a/materialize-sql/driver.go +++ b/materialize-sql/driver.go @@ -76,7 +76,10 @@ func (d *Driver) Validate(ctx context.Context, req *pm.Request_Validate) (*pm.Re return nil, fmt.Errorf("building endpoint: %w", err) } else if client, err = endpoint.NewClient(ctx, endpoint); err != nil { return nil, fmt.Errorf("creating client: %w", err) - } else if prereqErrs := client.PreReqs(ctx); prereqErrs.Len() != 0 { + } + defer client.Close() + + if prereqErrs := client.PreReqs(ctx); prereqErrs.Len() != 0 { return nil, cerrors.NewUserError(nil, prereqErrs.Error()) } else if loadedSpec, _, err = loadSpec(ctx, client, endpoint, req.Name); err != nil { return nil, fmt.Errorf("loading current applied materialization spec: %w", err) From 6f6b4f31cd8db0a6c7d5782fcebeba3eafe6bda0 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 22 Mar 2024 12:21:01 +0000 Subject: [PATCH 42/96] databricks: lower file upload concurrency databricks driver loads files to memory in full, causing OOM issues. This is a temporary measure until we fix this --- materialize-databricks/staged_file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/materialize-databricks/staged_file.go b/materialize-databricks/staged_file.go index 01f1a28df3..31df8f66fa 100644 --- a/materialize-databricks/staged_file.go +++ b/materialize-databricks/staged_file.go @@ -16,7 +16,7 @@ import ( ) const fileSizeLimit = 128 * 1024 * 1024 -const uploadConcurrency = 5 // that means we may use up to 1.28GB disk space +const uploadConcurrency = 3 // fileBuffer provides Close() for a *bufio.Writer writing to an *os.File. Close() will flush the // buffer and close the underlying file. From 55eee0b153ae05eb72e553659f1b4d989851892f Mon Sep 17 00:00:00 2001 From: Will Baker Date: Fri, 22 Mar 2024 10:10:50 -0400 Subject: [PATCH 43/96] materialize-snowflake: add temporary logging when encoding excessively large documents --- materialize-snowflake/snowflake.go | 4 ++-- materialize-snowflake/staged_file.go | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/materialize-snowflake/snowflake.go b/materialize-snowflake/snowflake.go index e2b931e047..b834433693 100644 --- a/materialize-snowflake/snowflake.go +++ b/materialize-snowflake/snowflake.go @@ -288,7 +288,7 @@ func (d *transactor) Load(it *m.LoadIterator, loaded func(int, json.RawMessage) return err } else if converted, err := b.target.ConvertKey(it.Key); err != nil { return fmt.Errorf("converting Load key: %w", err) - } else if err = b.load.stage.encodeRow(converted); err != nil { + } else if err = b.load.stage.encodeRow(converted, b.target.Source); err != nil { return fmt.Errorf("encoding Load key to scratch file: %w", err) } } @@ -422,7 +422,7 @@ func (d *transactor) Store(it *m.StoreIterator) (m.StartCommitFunc, error) { return nil, err } else if converted, err := b.target.ConvertAll(it.Key, it.Values, it.RawJSON); err != nil { return nil, fmt.Errorf("converting Store: %w", err) - } else if err = b.store.stage.encodeRow(converted); err != nil { + } else if err = b.store.stage.encodeRow(converted, b.target.Source); err != nil { return nil, fmt.Errorf("encoding Store to scratch file: %w", err) } } diff --git a/materialize-snowflake/staged_file.go b/materialize-snowflake/staged_file.go index 6dac0eb4e1..9dfb36ffbb 100644 --- a/materialize-snowflake/staged_file.go +++ b/materialize-snowflake/staged_file.go @@ -11,6 +11,7 @@ import ( "strings" sql "github.com/estuary/connectors/materialize-sql" + pf "github.com/estuary/flow/go/protocols/flow" "github.com/google/uuid" log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" @@ -155,7 +156,7 @@ func (f *stagedFile) start(ctx context.Context, db *stdsql.DB) error { return nil } -func (f *stagedFile) encodeRow(row []interface{}) error { +func (f *stagedFile) encodeRow(row []interface{}, collection pf.Collection) error { // May not have an encoder set yet if the previous encodeRow() resulted in flushing the current // file, or for the very first call to encodeRow(). if f.encoder == nil { @@ -164,10 +165,21 @@ func (f *stagedFile) encodeRow(row []interface{}) error { } } + initialWritten := f.encoder.Written() + if err := f.encoder.Encode(row); err != nil { return fmt.Errorf("encoding row: %w", err) } + // Temporary logging for encoding large documents that exceed Snowflake's 16MB limit. + delta := f.encoder.Written() - initialWritten + if delta > 16*1024*1024 { + log.WithFields(log.Fields{ + "collection": collection.String(), + "docSize": delta, + }).Warn("encoded a large document") + } + // Concurrently start the PUT process for this file if the current file has reached // fileSizeLimit. if f.encoder.Written() >= sql.DefaultFileSizeLimit { From 34c14314dcd165abcc66e313116b467ba0f5ff83 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Fri, 22 Mar 2024 11:09:14 -0400 Subject: [PATCH 44/96] materialize-snowflake: allow running integration test without private key set --- tests/materialize/materialize-snowflake/setup.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/materialize/materialize-snowflake/setup.sh b/tests/materialize/materialize-snowflake/setup.sh index 05954fee90..b4deb502af 100755 --- a/tests/materialize/materialize-snowflake/setup.sh +++ b/tests/materialize/materialize-snowflake/setup.sh @@ -14,8 +14,10 @@ export SNOWFLAKE_AUTH_TYPE="${SNOWFLAKE_AUTH_TYPE}" # if auth type is user_password export SNOWFLAKE_USER="${SNOWFLAKE_USER:-}" export SNOWFLAKE_PASSWORD="${SNOWFLAKE_PASSWORD:-}" -# if auth type is jwt -export SNOWFLAKE_PRIVATE_KEY="$(cat ${SNOWFLAKE_PRIVATE_KEY:-} | jq -sR . | sed -e 's/^"//' -e 's/"$//')" +if [ -z "${${SNOWFLAKE_PRIVATE_KEY+x}}" ]; then + # if auth type is jwt + export SNOWFLAKE_PRIVATE_KEY="$(cat ${SNOWFLAKE_PRIVATE_KEY:-} | jq -sR . | sed -e 's/^"//' -e 's/"$//')" +fi config_json_template='{ "host": "$SNOWFLAKE_HOST", From 82c3988e71da311745168b9cd75049a3564bbd00 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Fri, 22 Mar 2024 11:27:43 -0400 Subject: [PATCH 45/96] Revert "materialize-snowflake: add temporary logging when encoding excessively large documents" This reverts commit 55eee0b153ae05eb72e553659f1b4d989851892f. --- materialize-snowflake/snowflake.go | 4 ++-- materialize-snowflake/staged_file.go | 14 +------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/materialize-snowflake/snowflake.go b/materialize-snowflake/snowflake.go index b834433693..e2b931e047 100644 --- a/materialize-snowflake/snowflake.go +++ b/materialize-snowflake/snowflake.go @@ -288,7 +288,7 @@ func (d *transactor) Load(it *m.LoadIterator, loaded func(int, json.RawMessage) return err } else if converted, err := b.target.ConvertKey(it.Key); err != nil { return fmt.Errorf("converting Load key: %w", err) - } else if err = b.load.stage.encodeRow(converted, b.target.Source); err != nil { + } else if err = b.load.stage.encodeRow(converted); err != nil { return fmt.Errorf("encoding Load key to scratch file: %w", err) } } @@ -422,7 +422,7 @@ func (d *transactor) Store(it *m.StoreIterator) (m.StartCommitFunc, error) { return nil, err } else if converted, err := b.target.ConvertAll(it.Key, it.Values, it.RawJSON); err != nil { return nil, fmt.Errorf("converting Store: %w", err) - } else if err = b.store.stage.encodeRow(converted, b.target.Source); err != nil { + } else if err = b.store.stage.encodeRow(converted); err != nil { return nil, fmt.Errorf("encoding Store to scratch file: %w", err) } } diff --git a/materialize-snowflake/staged_file.go b/materialize-snowflake/staged_file.go index 9dfb36ffbb..6dac0eb4e1 100644 --- a/materialize-snowflake/staged_file.go +++ b/materialize-snowflake/staged_file.go @@ -11,7 +11,6 @@ import ( "strings" sql "github.com/estuary/connectors/materialize-sql" - pf "github.com/estuary/flow/go/protocols/flow" "github.com/google/uuid" log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" @@ -156,7 +155,7 @@ func (f *stagedFile) start(ctx context.Context, db *stdsql.DB) error { return nil } -func (f *stagedFile) encodeRow(row []interface{}, collection pf.Collection) error { +func (f *stagedFile) encodeRow(row []interface{}) error { // May not have an encoder set yet if the previous encodeRow() resulted in flushing the current // file, or for the very first call to encodeRow(). if f.encoder == nil { @@ -165,21 +164,10 @@ func (f *stagedFile) encodeRow(row []interface{}, collection pf.Collection) erro } } - initialWritten := f.encoder.Written() - if err := f.encoder.Encode(row); err != nil { return fmt.Errorf("encoding row: %w", err) } - // Temporary logging for encoding large documents that exceed Snowflake's 16MB limit. - delta := f.encoder.Written() - initialWritten - if delta > 16*1024*1024 { - log.WithFields(log.Fields{ - "collection": collection.String(), - "docSize": delta, - }).Warn("encoded a large document") - } - // Concurrently start the PUT process for this file if the current file has reached // fileSizeLimit. if f.encoder.Written() >= sql.DefaultFileSizeLimit { From 19c8da1294e0d20276e82646ecdc79ff679a4313 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Fri, 22 Mar 2024 12:01:28 -0400 Subject: [PATCH 46/96] source-gladly: add base_url advanced configuration option An alternate base URL is sometimes useful when connecting to a sandbox account, for example. --- source-gladly/source_gladly/api.py | 12 ++++--- source-gladly/source_gladly/models.py | 21 +++++++++++-- source-gladly/source_gladly/resources.py | 2 +- ...ly_tests_test_snapshots__spec__stdout.json | 31 +++++++++++++++++++ 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/source-gladly/source_gladly/api.py b/source-gladly/source_gladly/api.py index 16190d088e..2d625732e0 100644 --- a/source-gladly/source_gladly/api.py +++ b/source-gladly/source_gladly/api.py @@ -1,12 +1,12 @@ from datetime import UTC, datetime, timedelta from logging import Logger from typing import AsyncGenerator +from urllib.parse import urljoin -from estuary_cdk.http import HTTPSession from estuary_cdk.capture.common import LogCursor +from estuary_cdk.http import HTTPSession - -from .models import Event +from .models import EndpointConfig, Event # fetch_events gets the latest Events, starting with log_cursor and continuing to as close to the @@ -16,14 +16,16 @@ # ensures consistency of data while the connector is in normal operation. async def fetch_events( http: HTTPSession, - organization: str, + config: EndpointConfig, entity: str, log: Logger, log_cursor: LogCursor, ) -> AsyncGenerator[Event | LogCursor, None]: assert isinstance(log_cursor, datetime) - url = f"https://{organization}.gladly.com/api/v1/events" + base = config.advanced.base_url if config.advanced.base_url else f"https://{config.organization}.gladly.com" + url = urljoin(base, "api/v1/events") + params = { "startAt": log_cursor.isoformat(), "entities": entity, diff --git a/source-gladly/source_gladly/models.py b/source-gladly/source_gladly/models.py index 1745937b87..842000ca33 100644 --- a/source-gladly/source_gladly/models.py +++ b/source-gladly/source_gladly/models.py @@ -1,11 +1,14 @@ -from pydantic import AwareDatetime, BaseModel, Field +from typing import Optional from estuary_cdk.capture.common import ( - ConnectorState as GenericConnectorState, BaseDocument, ResourceConfig, ResourceState, ) +from estuary_cdk.capture.common import ( + ConnectorState as GenericConnectorState, +) +from pydantic import AwareDatetime, BaseModel, Field # Supported entity types for the Gladly Events API and the corresponding resource name. EVENTS_ENTITY_TYPES: list[tuple[str, str]] = [ @@ -34,6 +37,20 @@ class EndpointConfig(BaseModel): json_schema_extra={"secret": True}, ) + class Advanced(BaseModel): + base_url: Optional[str] = Field( + None, + description="Base URL to use for connecting to Gladly. May be useful if you are connecting to a sandbox account that does not use a URL in the form of 'https://{organization}.gladly.com'", + title="Base URL", + ) + + advanced: Advanced = Field( + default_factory=Advanced, + title="Advanced Config", + description="Advanced settings for the connector.", + json_schema_extra={"advanced": True}, + ) + ConnectorState = GenericConnectorState[ResourceState] diff --git a/source-gladly/source_gladly/resources.py b/source-gladly/source_gladly/resources.py index 45ad08d512..14f1965425 100644 --- a/source-gladly/source_gladly/resources.py +++ b/source-gladly/source_gladly/resources.py @@ -44,7 +44,7 @@ def open( fetch_changes=functools.partial( fetch_events, http, - config.organization, + config, entity, ), ) diff --git a/source-gladly/tests/snapshots/source_gladly_tests_test_snapshots__spec__stdout.json b/source-gladly/tests/snapshots/source_gladly_tests_test_snapshots__spec__stdout.json index b9d41f59ed..27da22de2a 100644 --- a/source-gladly/tests/snapshots/source_gladly_tests_test_snapshots__spec__stdout.json +++ b/source-gladly/tests/snapshots/source_gladly_tests_test_snapshots__spec__stdout.json @@ -2,6 +2,27 @@ { "protocol": 3032023, "configSchema": { + "$defs": { + "Advanced": { + "properties": { + "base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Base URL to use for connecting to Gladly. May be useful if you are connecting to a sandbox account that does not use a URL in the form of 'https://{organization}.gladly.com'", + "title": "Base URL" + } + }, + "title": "Advanced", + "type": "object" + } + }, "properties": { "organization": { "description": "Organization to Request Data From", @@ -18,6 +39,16 @@ "secret": true, "title": "API Token", "type": "string" + }, + "advanced": { + "advanced": true, + "allOf": [ + { + "$ref": "#/$defs/Advanced" + } + ], + "description": "Advanced settings for the connector.", + "title": "Advanced Config" } }, "required": [ From c37bd4fcd0174a42c67ba73f4b966df13e88d11e Mon Sep 17 00:00:00 2001 From: Will Baker Date: Fri, 22 Mar 2024 12:32:25 -0400 Subject: [PATCH 47/96] source-gladly(refactor): add init file to tests folder Refactor-only commit that adds the `__init__.py` file to the tests folder so that `pytest` can be run from the connector directory, as we have done with other python connectors. Renames the snapshot files, but other than that nothing is changed. --- source-gladly/tests/__init__.py | 0 ...ts__discover__stdout.json => snapshots__discover__stdout.json} | 0 ..._snapshots__spec__stdout.json => snapshots__spec__stdout.json} | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 source-gladly/tests/__init__.py rename source-gladly/tests/snapshots/{source_gladly_tests_test_snapshots__discover__stdout.json => snapshots__discover__stdout.json} (100%) rename source-gladly/tests/snapshots/{source_gladly_tests_test_snapshots__spec__stdout.json => snapshots__spec__stdout.json} (100%) diff --git a/source-gladly/tests/__init__.py b/source-gladly/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source-gladly/tests/snapshots/source_gladly_tests_test_snapshots__discover__stdout.json b/source-gladly/tests/snapshots/snapshots__discover__stdout.json similarity index 100% rename from source-gladly/tests/snapshots/source_gladly_tests_test_snapshots__discover__stdout.json rename to source-gladly/tests/snapshots/snapshots__discover__stdout.json diff --git a/source-gladly/tests/snapshots/source_gladly_tests_test_snapshots__spec__stdout.json b/source-gladly/tests/snapshots/snapshots__spec__stdout.json similarity index 100% rename from source-gladly/tests/snapshots/source_gladly_tests_test_snapshots__spec__stdout.json rename to source-gladly/tests/snapshots/snapshots__spec__stdout.json From efd7836ffcc5d361f7d25b5228b526659ee4e910 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Fri, 22 Mar 2024 18:02:22 -0500 Subject: [PATCH 48/96] source-redshift-batch: Fix handling of column nullability Currently most discovery flows are broken because we treat columns as always being non-null due to a dumb oversight in how the boolean nullability value is (not) plumbed through to schema generation, so then the moment you actually start capturing data you get a ton of schema violations wherever nulls actually exist. This commit fixes that oversight. --- .../.snapshots/TestBasicCapture-Discovery | 5 +- .../.snapshots/TestBasicDatatypes-Discovery | 31 +++++-- .../.snapshots/TestKeyDiscovery | 5 +- .../.snapshots/TestKeylessDiscovery | 46 ++++++++-- .../.snapshots/TestSchemaFilter-FilteredIn | 5 +- .../.snapshots/TestSchemaFilter-Unfiltered | 5 +- source-redshift-batch/driver.go | 89 ++++++++++--------- source-redshift-batch/main_test.go | 2 +- 8 files changed, 123 insertions(+), 65 deletions(-) diff --git a/source-redshift-batch/.snapshots/TestBasicCapture-Discovery b/source-redshift-batch/.snapshots/TestBasicCapture-Discovery index 43e55f91b7..46f6e555de 100644 --- a/source-redshift-batch/.snapshots/TestBasicCapture-Discovery +++ b/source-redshift-batch/.snapshots/TestBasicCapture-Discovery @@ -39,7 +39,10 @@ Binding 0: ] }, "data": { - "type": "string" + "type": [ + "string", + "null" + ] }, "id": { "type": "integer" diff --git a/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery b/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery index 3fcb2b5f61..0b4b3bec26 100644 --- a/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery +++ b/source-redshift-batch/.snapshots/TestBasicDatatypes-Discovery @@ -39,22 +39,37 @@ Binding 0: ] }, "a_bool": { - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "a_date": { - "type": "string", - "format": "date-time" + "format": "date-time", + "type": [ + "string", + "null" + ] }, "a_real": { - "type": "number" + "type": [ + "number", + "null" + ] }, "a_ts": { - "type": "string", - "format": "date-time" + "format": "date-time", + "type": [ + "string", + "null" + ] }, "a_tstz": { - "type": "string", - "format": "date-time" + "format": "date-time", + "type": [ + "string", + "null" + ] }, "id": { "type": "integer" diff --git a/source-redshift-batch/.snapshots/TestKeyDiscovery b/source-redshift-batch/.snapshots/TestKeyDiscovery index 08f4957a24..b9441fc6df 100644 --- a/source-redshift-batch/.snapshots/TestKeyDiscovery +++ b/source-redshift-batch/.snapshots/TestKeyDiscovery @@ -43,7 +43,10 @@ Binding 0: ] }, "data": { - "type": "string" + "type": [ + "string", + "null" + ] }, "k_bigint": { "type": "integer" diff --git a/source-redshift-batch/.snapshots/TestKeylessDiscovery b/source-redshift-batch/.snapshots/TestKeylessDiscovery index 59e20c095a..6b6a659280 100644 --- a/source-redshift-batch/.snapshots/TestKeylessDiscovery +++ b/source-redshift-batch/.snapshots/TestKeylessDiscovery @@ -38,30 +38,60 @@ Binding 0: ] }, "v_bigint": { - "type": "integer" + "type": [ + "integer", + "null" + ] }, "v_bool": { - "type": "boolean" + "type": [ + "boolean", + "null" + ] }, "v_int": { + "type": [ + "integer", + "null" + ] + }, + "v_int_notnull": { "type": "integer" }, "v_smallint": { - "type": "integer" + "type": [ + "integer", + "null" + ] }, "v_str": { - "type": "string" + "type": [ + "string", + "null" + ] }, "v_text": { + "type": [ + "string", + "null" + ] + }, + "v_text_notnull": { "type": "string" }, "v_ts": { - "type": "string", - "format": "date-time" + "format": "date-time", + "type": [ + "string", + "null" + ] }, "v_tstz": { - "type": "string", - "format": "date-time" + "format": "date-time", + "type": [ + "string", + "null" + ] } } }, diff --git a/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn b/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn index cbb7ec02ac..0478f0e132 100644 --- a/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn +++ b/source-redshift-batch/.snapshots/TestSchemaFilter-FilteredIn @@ -39,7 +39,10 @@ Binding 0: ] }, "data": { - "type": "string" + "type": [ + "string", + "null" + ] }, "id": { "type": "integer" diff --git a/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered b/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered index cbb7ec02ac..0478f0e132 100644 --- a/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered +++ b/source-redshift-batch/.snapshots/TestSchemaFilter-Unfiltered @@ -39,7 +39,10 @@ Binding 0: ] }, "data": { - "type": "string" + "type": [ + "string", + "null" + ] }, "id": { "type": "integer" diff --git a/source-redshift-batch/driver.go b/source-redshift-batch/driver.go index 56e0eb44e2..c73ff2f334 100644 --- a/source-redshift-batch/driver.go +++ b/source-redshift-batch/driver.go @@ -414,19 +414,20 @@ func discoverColumns(ctx context.Context, db *sql.DB, discoverSchemas []string) } // Decode column type information into a usable form - var dataType columnType + var dataType basicColumnType switch typeType { case "e": // enum values are captured as strings - dataType = &basicColumnType{jsonType: "string"} + dataType = basicColumnType{jsonType: "string"} case "r", "m": // ranges and multiranges are captured as strings - dataType = &basicColumnType{jsonType: "string"} + dataType = basicColumnType{jsonType: "string"} default: var ok bool dataType, ok = databaseTypeToJSON[typeName] if !ok { - dataType = &basicColumnType{description: fmt.Sprintf("using catch-all schema for unknown type %q", typeName)} + dataType = basicColumnType{description: fmt.Sprintf("using catch-all schema for unknown type %q", typeName)} } } + dataType.nullable = isNullable columns = append(columns, &discoveredColumn{ Schema: tableSchema, @@ -434,7 +435,7 @@ func discoverColumns(ctx context.Context, db *sql.DB, discoverSchemas []string) Name: columnName, Index: columnIndex, IsNullable: isNullable, - DataType: dataType, + DataType: &dataType, }) } return columns, nil @@ -499,51 +500,51 @@ func discoverPrimaryKeys(ctx context.Context, db *sql.DB, discoverSchemas []stri return keys, nil } -var databaseTypeToJSON = map[string]columnType{ - "bool": &basicColumnType{jsonType: "boolean"}, +var databaseTypeToJSON = map[string]basicColumnType{ + "bool": {jsonType: "boolean"}, - "int2": &basicColumnType{jsonType: "integer"}, - "int4": &basicColumnType{jsonType: "integer"}, - "int8": &basicColumnType{jsonType: "integer"}, + "int2": {jsonType: "integer"}, + "int4": {jsonType: "integer"}, + "int8": {jsonType: "integer"}, - "numeric": &basicColumnType{jsonType: "string", format: "number"}, - "float4": &basicColumnType{jsonType: "number"}, - "float8": &basicColumnType{jsonType: "number"}, + "numeric": {jsonType: "string", format: "number"}, + "float4": {jsonType: "number"}, + "float8": {jsonType: "number"}, - "varchar": &basicColumnType{jsonType: "string"}, - "bpchar": &basicColumnType{jsonType: "string"}, - "text": &basicColumnType{jsonType: "string"}, - "bytea": &basicColumnType{jsonType: "string", contentEncoding: "base64"}, - "xml": &basicColumnType{jsonType: "string"}, - "bit": &basicColumnType{jsonType: "string"}, - "varbit": &basicColumnType{jsonType: "string"}, + "varchar": {jsonType: "string"}, + "bpchar": {jsonType: "string"}, + "text": {jsonType: "string"}, + "bytea": {jsonType: "string", contentEncoding: "base64"}, + "xml": {jsonType: "string"}, + "bit": {jsonType: "string"}, + "varbit": {jsonType: "string"}, - "json": &basicColumnType{}, - "jsonb": &basicColumnType{}, - "jsonpath": &basicColumnType{jsonType: "string"}, + "json": {}, + "jsonb": {}, + "jsonpath": {jsonType: "string"}, // Domain-Specific Types - "date": &basicColumnType{jsonType: "string", format: "date-time"}, - "timestamp": &basicColumnType{jsonType: "string", format: "date-time"}, - "timestamptz": &basicColumnType{jsonType: "string", format: "date-time"}, - "time": &basicColumnType{jsonType: "integer"}, - "timetz": &basicColumnType{jsonType: "string", format: "time"}, - "interval": &basicColumnType{jsonType: "string"}, - "money": &basicColumnType{jsonType: "string"}, - "point": &basicColumnType{jsonType: "string"}, - "line": &basicColumnType{jsonType: "string"}, - "lseg": &basicColumnType{jsonType: "string"}, - "box": &basicColumnType{jsonType: "string"}, - "path": &basicColumnType{jsonType: "string"}, - "polygon": &basicColumnType{jsonType: "string"}, - "circle": &basicColumnType{jsonType: "string"}, - "inet": &basicColumnType{jsonType: "string"}, - "cidr": &basicColumnType{jsonType: "string"}, - "macaddr": &basicColumnType{jsonType: "string"}, - "macaddr8": &basicColumnType{jsonType: "string"}, - "tsvector": &basicColumnType{jsonType: "string"}, - "tsquery": &basicColumnType{jsonType: "string"}, - "uuid": &basicColumnType{jsonType: "string", format: "uuid"}, + "date": {jsonType: "string", format: "date-time"}, + "timestamp": {jsonType: "string", format: "date-time"}, + "timestamptz": {jsonType: "string", format: "date-time"}, + "time": {jsonType: "integer"}, + "timetz": {jsonType: "string", format: "time"}, + "interval": {jsonType: "string"}, + "money": {jsonType: "string"}, + "point": {jsonType: "string"}, + "line": {jsonType: "string"}, + "lseg": {jsonType: "string"}, + "box": {jsonType: "string"}, + "path": {jsonType: "string"}, + "polygon": {jsonType: "string"}, + "circle": {jsonType: "string"}, + "inet": {jsonType: "string"}, + "cidr": {jsonType: "string"}, + "macaddr": {jsonType: "string"}, + "macaddr8": {jsonType: "string"}, + "tsvector": {jsonType: "string"}, + "tsquery": {jsonType: "string"}, + "uuid": {jsonType: "string", format: "uuid"}, } var catalogNameSanitizerRe = regexp.MustCompile(`(?i)[^a-z0-9\-_.]`) diff --git a/source-redshift-batch/main_test.go b/source-redshift-batch/main_test.go index 5bbfb56325..bfe93f9a88 100644 --- a/source-redshift-batch/main_test.go +++ b/source-redshift-batch/main_test.go @@ -202,7 +202,7 @@ func TestKeylessDiscovery(t *testing.T) { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) t.Cleanup(func() { executeControlQuery(ctx, t, control, fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)) }) - executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(v_smallint SMALLINT, v_int INTEGER, v_bigint BIGINT, v_bool BOOLEAN, v_str VARCHAR(8), v_ts TIMESTAMP, v_tstz TIMESTAMP WITH TIME ZONE, v_text TEXT)", tableName)) + executeControlQuery(ctx, t, control, fmt.Sprintf("CREATE TABLE %s(v_smallint SMALLINT, v_int INTEGER, v_bigint BIGINT, v_bool BOOLEAN, v_str VARCHAR(8), v_ts TIMESTAMP, v_tstz TIMESTAMP WITH TIME ZONE, v_text TEXT, v_int_notnull INTEGER NOT NULL, v_text_notnull TEXT NOT NULL)", tableName)) cs.EndpointSpec.(*Config).Advanced.DiscoverSchemas = []string{"test"} snapshotBindings(t, discoverStreams(ctx, t, cs, regexp.MustCompile(uniqueID))) From 2eea5bca0aafb240475b6837385ed23bb1d339f3 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Thu, 21 Mar 2024 10:31:48 -0400 Subject: [PATCH 49/96] import: source-notion through airbyte @ 8c862a8 Remote Repo URL: git@github.com:airbytehq/airbyte.git Source name: 8c862a8 Source Commit ID: 8c862a8013150cfee238c379e66ff9e84550da6b Source Repo Prefix: airbyte-integrations/connectors/source-notion/ Import Path: source-notion/ License Type: MIT License Path: airbyte-integrations/connectors/source-notion/metadata.yaml git-merge-subpath: 8c862a8013150cfee238c379e66ff9e84550da6b airbyte-integrations/connectors/source-notion source-notion --- source-notion/.dockerignore | 7 + source-notion/README.md | 91 ++ source-notion/acceptance-test-config.yml | 39 + source-notion/bootstrap.md | 32 + source-notion/icon.svg | 1 + source-notion/integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 35 + source-notion/integration_tests/acceptance.py | 14 + source-notion/integration_tests/catalog.json | 38 + .../integration_tests/configured_catalog.json | 71 ++ .../integration_tests/expected_records.jsonl | 15 + .../incremental_catalog.json | 46 + .../integration_tests/invalid_config.json | 9 + .../integration_tests/sample_config.json | 7 + .../sample_config_oauth.json | 9 + .../integration_tests/sample_state.json | 46 + source-notion/main.py | 8 + source-notion/metadata.yaml | 49 + source-notion/poetry.lock | 1032 +++++++++++++++++ source-notion/pyproject.toml | 30 + source-notion/source_notion/__init__.py | 8 + source-notion/source_notion/run.py | 14 + .../source_notion/schemas/blocks.json | 289 +++++ .../source_notion/schemas/comments.json | 93 ++ .../source_notion/schemas/databases.json | 555 +++++++++ .../source_notion/schemas/pages.json | 330 ++++++ .../source_notion/schemas/shared/child.json | 8 + .../source_notion/schemas/shared/date.json | 10 + .../source_notion/schemas/shared/emoji.json | 13 + .../source_notion/schemas/shared/file.json | 38 + .../source_notion/schemas/shared/heading.json | 10 + .../source_notion/schemas/shared/icon.json | 4 + .../source_notion/schemas/shared/options.json | 27 + .../source_notion/schemas/shared/parent.json | 22 + .../schemas/shared/rich_text.json | 89 ++ .../schemas/shared/text_element.json | 10 + .../source_notion/schemas/shared/title.json | 62 + .../source_notion/schemas/shared/user.json | 76 ++ .../source_notion/schemas/users.json | 4 + source-notion/source_notion/source.py | 98 ++ source-notion/source_notion/spec.json | 119 ++ source-notion/source_notion/streams.py | 453 ++++++++ source-notion/source_notion/utils.py | 63 + source-notion/unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 403 +++++++ source-notion/unit_tests/test_source.py | 86 ++ source-notion/unit_tests/test_streams.py | 352 ++++++ 47 files changed, 4821 insertions(+) create mode 100644 source-notion/.dockerignore create mode 100644 source-notion/README.md create mode 100644 source-notion/acceptance-test-config.yml create mode 100644 source-notion/bootstrap.md create mode 100644 source-notion/icon.svg create mode 100644 source-notion/integration_tests/__init__.py create mode 100644 source-notion/integration_tests/abnormal_state.json create mode 100644 source-notion/integration_tests/acceptance.py create mode 100644 source-notion/integration_tests/catalog.json create mode 100644 source-notion/integration_tests/configured_catalog.json create mode 100644 source-notion/integration_tests/expected_records.jsonl create mode 100644 source-notion/integration_tests/incremental_catalog.json create mode 100644 source-notion/integration_tests/invalid_config.json create mode 100644 source-notion/integration_tests/sample_config.json create mode 100644 source-notion/integration_tests/sample_config_oauth.json create mode 100644 source-notion/integration_tests/sample_state.json create mode 100644 source-notion/main.py create mode 100644 source-notion/metadata.yaml create mode 100644 source-notion/poetry.lock create mode 100644 source-notion/pyproject.toml create mode 100644 source-notion/source_notion/__init__.py create mode 100644 source-notion/source_notion/run.py create mode 100644 source-notion/source_notion/schemas/blocks.json create mode 100644 source-notion/source_notion/schemas/comments.json create mode 100644 source-notion/source_notion/schemas/databases.json create mode 100644 source-notion/source_notion/schemas/pages.json create mode 100644 source-notion/source_notion/schemas/shared/child.json create mode 100644 source-notion/source_notion/schemas/shared/date.json create mode 100644 source-notion/source_notion/schemas/shared/emoji.json create mode 100644 source-notion/source_notion/schemas/shared/file.json create mode 100644 source-notion/source_notion/schemas/shared/heading.json create mode 100644 source-notion/source_notion/schemas/shared/icon.json create mode 100644 source-notion/source_notion/schemas/shared/options.json create mode 100644 source-notion/source_notion/schemas/shared/parent.json create mode 100644 source-notion/source_notion/schemas/shared/rich_text.json create mode 100644 source-notion/source_notion/schemas/shared/text_element.json create mode 100644 source-notion/source_notion/schemas/shared/title.json create mode 100644 source-notion/source_notion/schemas/shared/user.json create mode 100644 source-notion/source_notion/schemas/users.json create mode 100644 source-notion/source_notion/source.py create mode 100644 source-notion/source_notion/spec.json create mode 100644 source-notion/source_notion/streams.py create mode 100644 source-notion/source_notion/utils.py create mode 100644 source-notion/unit_tests/__init__.py create mode 100644 source-notion/unit_tests/test_incremental_streams.py create mode 100644 source-notion/unit_tests/test_source.py create mode 100644 source-notion/unit_tests/test_streams.py diff --git a/source-notion/.dockerignore b/source-notion/.dockerignore new file mode 100644 index 0000000000..955cfbdc08 --- /dev/null +++ b/source-notion/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_notion +!setup.py +!secrets diff --git a/source-notion/README.md b/source-notion/README.md new file mode 100644 index 0000000000..13b0ba9f2b --- /dev/null +++ b/source-notion/README.md @@ -0,0 +1,91 @@ +# Notion source connector + + +This is the repository for the Notion source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/notion). + +## Local development + +### Prerequisites +* Python (~=3.9) +* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) + + +### Installing the connector +From this connector directory, run: +```bash +poetry install --with dev +``` + + +### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/notion) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_notion/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + + +### Locally running the connector +``` +poetry run source-notion spec +poetry run source-notion check --config secrets/config.json +poetry run source-notion discover --config secrets/config.json +poetry run source-notion read --config secrets/config.json --catalog sample_files/configured_catalog.json +``` + +### Running unit tests +To run unit tests locally, from the connector directory run: +``` +poetry run pytest unit_tests +``` + +### Building the docker image +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: +```bash +airbyte-ci connectors --name=source-notion build +``` + +An image will be available on your host with the tag `airbyte/source-notion:dev`. + + +### Running as a docker container +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-notion:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-notion:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-notion:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-notion:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +### Running our CI test suite +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-notion test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +### Dependency Management +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-notion test` +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` +3. Make sure the `metadata.yaml` content is up to date. +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/notion.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/source-notion/acceptance-test-config.yml b/source-notion/acceptance-test-config.yml new file mode 100644 index 0000000000..ab14b89dc5 --- /dev/null +++ b/source-notion/acceptance-test-config.yml @@ -0,0 +1,39 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-notion:dev +test_strictness_level: high +acceptance_tests: + spec: + tests: + - spec_path: "source_notion/spec.json" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + # 2.0.8 introduces a fix to _blocks.properties.table_row.cells, + # which was incorrectly added in 2.0.0 as an object array instead of an array of object arrays. + disable_for_version: 2.0.7 + basic_read: + tests: + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + fail_on_extra_columns: true + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/source-notion/bootstrap.md b/source-notion/bootstrap.md new file mode 100644 index 0000000000..6d492039f8 --- /dev/null +++ b/source-notion/bootstrap.md @@ -0,0 +1,32 @@ +# Notion + +## Overview + +Notion is an application that provides components such as notes, databases, kanban boards, wikis, calendars and reminders. Notion REST API allows a developer to retrieve pages, databases, blocks, and users on the Notion platform. + +## Endpoints + +Notion API consists of three endpoints which can be extracted data from: + +1. **User**: The User object represents a user in a Notion workspace. Users include guests, full workspace members, and bots. +2. **Block**: A block object represents content within Notion. Blocks can be text, lists, media, and more. Page and database is also a type of block. +3. **Search**: This endpoint is used to get list of pages and databases. + +## Quick Notes + +- Notion stores content in hierarchy, each node is called a 'block'. Block is a generic term which can be text, lists, media, even page and database are also block. + +- Due to this hierarchical structure, we use recursive request to get the full list of blocks. + +- Pages and databases can be extracted from the `Search` endpoint separately, so they are excluded from the block list request. + +- Airbyte CDK doesn't support recursive schema, so some elements of the block schema which can be recursive are replaced with empty objects. + +- Page and database must grant permission to the internal integration, otherwise API cannot extract data from them. See [https://developers.notion.com/docs/authorization#authorizing-internal-integrations](https://developers.notion.com/docs/authorization#authorizing-internal-integrations) + +- Rate limiting is a standard exponential backoff when a 429 HTTP status code returned. The rate limit for incoming requests is an average of 3 requests per second. Some bursts beyond the average rate are allowed. Notion API also has size limit, see [https://developers.notion.com/reference/errors#request-limits](https://developers.notion.com/reference/errors#request-limits) + +## API Reference + +The API reference documents: [https://developers.notion.com/reference/intro](https://developers.notion.com/reference) + diff --git a/source-notion/icon.svg b/source-notion/icon.svg new file mode 100644 index 0000000000..8b34d594bd --- /dev/null +++ b/source-notion/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/source-notion/integration_tests/__init__.py b/source-notion/integration_tests/__init__.py new file mode 100644 index 0000000000..46b7376756 --- /dev/null +++ b/source-notion/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/source-notion/integration_tests/abnormal_state.json b/source-notion/integration_tests/abnormal_state.json new file mode 100644 index 0000000000..b791f731fb --- /dev/null +++ b/source-notion/integration_tests/abnormal_state.json @@ -0,0 +1,35 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "last_edited_time": "2099-10-10T04:40:00.000Z" + }, + "stream_descriptor": { + "name": "databases" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last_edited_time": "2099-10-10T04:40:00.000Z" + }, + "stream_descriptor": { + "name": "pages" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last_edited_time": "2099-10-10T04:00:00.000Z" + }, + "stream_descriptor": { + "name": "blocks" + } + } + } +] diff --git a/source-notion/integration_tests/acceptance.py b/source-notion/integration_tests/acceptance.py new file mode 100644 index 0000000000..82823254d2 --- /dev/null +++ b/source-notion/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/source-notion/integration_tests/catalog.json b/source-notion/integration_tests/catalog.json new file mode 100644 index 0000000000..3b76e23aa3 --- /dev/null +++ b/source-notion/integration_tests/catalog.json @@ -0,0 +1,38 @@ +{ + "streams": [ + { + "name": "users", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "json_schema": {} + }, + { + "name": "databases", + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": "last_edited_time", + "json_schema": {} + }, + { + "name": "pages", + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": "last_edited_time", + "json_schema": {} + }, + { + "name": "blocks", + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": "last_edited_time", + "json_schema": {} + }, + { + "name": "comments", + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": "page_last_edited_time", + "json_schema": {} + } + ] +} diff --git a/source-notion/integration_tests/configured_catalog.json b/source-notion/integration_tests/configured_catalog.json new file mode 100644 index 0000000000..d0fa294445 --- /dev/null +++ b/source-notion/integration_tests/configured_catalog.json @@ -0,0 +1,71 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "default_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + }, + { + "stream": { + "name": "databases", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_primary_key": [["id"]], + "default_cursor_field": ["last_edited_time"] + }, + "primary_key": [["id"]], + "cursor_field": ["last_edited_time"], + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "pages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_primary_key": [["id"]], + "default_cursor_field": ["last_edited_time"] + }, + "primary_key": [["id"]], + "cursor_field": ["last_edited_time"], + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "blocks", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_primary_key": [["id"]], + "default_cursor_field": ["last_edited_time"] + }, + "primary_key": [["id"]], + "cursor_field": ["last_edited_time"], + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_primary_key": [["id"]], + "default_cursor_field": ["page_last_edited_time"] + }, + "primary_key": [["id"]], + "cursor_field": ["page_last_edited_time"], + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/source-notion/integration_tests/expected_records.jsonl b/source-notion/integration_tests/expected_records.jsonl new file mode 100644 index 0000000000..9989f84792 --- /dev/null +++ b/source-notion/integration_tests/expected_records.jsonl @@ -0,0 +1,15 @@ +{"stream": "users", "data": {"object": "user", "id": "5612c094-99ec-4ba3-ac7f-df8d84c8d6be", "name": "Sherif Nada", "avatar_url": "https://s3-us-west-2.amazonaws.com/public.notion-static.com/305f7efc-2862-4342-ba99-5023f3e34717/6246757.png", "type": "person", "person": {"email": "sherif@airbyte.io"}}, "emitted_at": 1697023279924} +{"stream": "users", "data": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}, "emitted_at": 1697023279925} +{"stream": "users", "data": {"object": "user", "id": "c1ff0160-b2af-497a-aab7-8b61e625e4e3", "name": "Gil Cho", "avatar_url": "https://lh3.googleusercontent.com/a/ALm5wu0ElXfvy3YfVUyRn-aB9EZy5AZ1ougHuNyCGmO2=s100", "type": "person", "person": {"email": "gil@airbyte.io"}}, "emitted_at": 1697023279925} +{"stream": "databases", "data": {"object": "database", "id": "b75d2e55-cc80-4afa-a273-c78178ac6b3f", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\ude4b"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Engineering Directory ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Engineering Directory ", "href": null}], "description": [{"type": "text", "text": {"content": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "href": null}, {"type": "text", "text": {"content": "Notion databases", "link": {"url": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Notion databases", "href": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}, {"type": "text", "text": {"content": ".", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": ".", "href": null}], "is_inline": false, "properties": [{"name": "Date Added", "value": {"id": "%2Fkv%22", "name": "Date Added", "type": "created_time", "created_time": {}}}, {"name": "Notes", "value": {"id": "mq%22D", "name": "Notes", "type": "rich_text", "rich_text": {}}}, {"name": "Person", "value": {"id": "uiZ%26", "name": "Person", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "block_id", "block_id": "b81f8caf-3ec4-4455-9a0b-25c2bd3b60cb"}, "url": "https://www.notion.so/b75d2e55cc804afaa273c78178ac6b3f", "public_url": null, "archived": false}, "emitted_at": 1708341487319} +{"stream": "databases", "data": {"object": "database", "id": "fbff7d4e-eca4-4432-91e6-ec64ba4b5a98", "cover": null, "icon": null, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Questions", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Questions", "href": null}], "description": [], "is_inline": true, "properties": [{"name": "Difficulty", "value": {"id": "'i6%2F", "name": "Difficulty", "type": "select", "select": {"options": [{"id": "f00068b9-7612-45da-91ad-1a7b1d259375", "name": "Easy", "color": "green", "description": null}, {"id": "8e244bfe-d4c7-48c5-9088-ffd6926b4ba0", "name": "Medium", "color": "yellow", "description": null}, {"id": "9ab57ef4-eab1-4b20-a502-047610b5c97d", "name": "Hard", "color": "red", "description": null}]}}}, {"name": "Skills", "value": {"id": "K%3AtR", "name": "Skills", "type": "multi_select", "multi_select": {"options": [{"id": "72f4d134-a773-48c1-ba3d-b529f55c6818", "name": "Front end", "color": "default", "description": null}, {"id": "c20f5d57-3e35-4b39-b556-05071203cc1a", "name": "Backend", "color": "default", "description": null}, {"id": "31d5735c-d6ba-4bd7-940f-bdcb36091c02", "name": "Architecture", "color": "default", "description": null}, {"id": "0398de54-af68-4c3a-9953-3788e8eaadbf", "name": "Algorithms", "color": "default", "description": null}, {"id": "df9dff09-7dea-4409-a10f-b5e2b546ad94", "name": "Data Structures", "color": "default", "description": null}]}}}, {"name": "Question Name", "value": {"id": "title", "name": "Question Name", "type": "title", "title": {}}}], "parent": {"type": "page_id", "page_id": "4999109d-1b7b-41a2-abb4-84f6b961ee74"}, "url": "https://www.notion.so/fbff7d4eeca4443291e6ec64ba4b5a98", "public_url": null, "archived": false}, "emitted_at": 1708341487319} +{"stream": "databases", "data": {"object": "database", "id": "9b1ce91e-a93a-437c-8c92-81083cd98540", "cover": null, "icon": {"type": "emoji", "emoji": "\u270f\ufe0f"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "last_edited_time": "2023-09-13T00:06:00.000Z", "title": [{"type": "text", "text": {"content": "Meeting Notes", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Meeting Notes", "href": null}], "description": [{"type": "text", "text": {"content": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "href": null}, {"type": "text", "text": {"content": "\u2193", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "\u2193", "href": null}, {"type": "text", "text": {"content": " Click ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " Click ", "href": null}, {"type": "text", "text": {"content": "List View", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": true, "color": "default"}, "plain_text": "List View", "href": null}, {"type": "text", "text": {"content": " to create and see other views, including a board organized by meeting type.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " to create and see other views, including a board organized by meeting type.", "href": null}], "is_inline": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "name": "Last Edited Time", "type": "last_edited_time", "last_edited_time": {}}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "name": "Created By", "type": "created_by", "created_by": {}}}, {"name": "Created", "value": {"id": "Ird4", "name": "Created", "type": "created_time", "created_time": {}}}, {"name": "Type", "value": {"id": "_%7B%5C7", "name": "Type", "type": "select", "select": {"options": [{"id": "3a8fd64c-899d-4c39-ba97-ac4f565d6e94", "name": "Post-mortem", "color": "red", "description": null}, {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green", "description": null}, {"id": "8ee247a9-cb60-430a-9ea6-d5c053253334", "name": "Weekly Sync", "color": "blue", "description": null}, {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow", "description": null}, {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange", "description": null}]}}}, {"name": "Participants", "value": {"id": "b%3AeA", "name": "Participants", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/9b1ce91ea93a437c8c9281083cd98540", "public_url": null, "archived": false}, "emitted_at": 1708341487319} +{"stream": "pages", "data": {"object": "page", "id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Daily Standup", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Daily Standup", "href": null}]}}], "url": "https://www.notion.so/Daily-Standup-39a69b4e7cc24f7aa656dd128f3ce855", "public_url": null}, "emitted_at": 1697023284463} +{"stream": "pages", "data": {"object": "page", "id": "621d3dc4-55fe-46ce-a3ff-83da06e5f9fb", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Sprint Planning ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Sprint Planning ", "href": null}]}}], "url": "https://www.notion.so/Sprint-Planning-621d3dc455fe46cea3ff83da06e5f9fb", "public_url": null}, "emitted_at": 1697023284465} +{"stream": "pages", "data": {"object": "page", "id": "6eb2dedc-8b88-486c-8648-d1878bafb106", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Ad Hoc Meeting", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Ad Hoc Meeting", "href": null}]}}], "url": "https://www.notion.so/Ad-Hoc-Meeting-6eb2dedc8b88486c8648d1878bafb106", "public_url": null}, "emitted_at": 1697023284465} +{"stream": "blocks", "data": {"object": "block", "id": "b54364a0-ff86-45ba-b78e-a32018446a3f", "parent": {"type": "page_id", "page_id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "callout", "callout": {"rich_text": [{"type": "text", "text": {"content": "Change the title to include the date.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Change the title to include the date.", "href": null}], "icon": {"type": "emoji", "emoji": "\ud83d\udca1"}, "color": "gray_background"}}, "emitted_at": 1697023288683} +{"stream": "blocks", "data": {"object": "block", "id": "c6608513-db08-4411-8ec6-e343580cbf84", "parent": {"type": "page_id", "page_id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"rich_text": [{"type": "text", "text": {"content": "What did we do yesterday?", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "What did we do yesterday?", "href": null}], "is_toggleable": false, "color": "default"}}, "emitted_at": 1697023288684} +{"stream": "blocks", "data": {"object": "block", "id": "ffb233aa-59da-4d04-9cc8-e6b767bd1a85", "parent": {"type": "page_id", "page_id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "bulleted_list_item", "bulleted_list_item": {"rich_text": [], "color": "default"}}, "emitted_at": 1697023288684} +{"stream": "comments", "data": {"object": "comment", "id": "e2dd2530-3ef1-4a27-83fb-3f16400b9838", "parent": {"type": "page_id", "page_id": "a55d276e-4bc2-4fcc-9fb3-e60b867c86e7"}, "discussion_id": "15e3cffe-3c9d-4ef2-87b1-86c46f85a205", "created_time": "2023-10-10T13:52:00.000Z", "last_edited_time": "2023-10-10T13:52:00.000Z", "created_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "rich_text": [{"type": "text", "text": {"content": "Gathered voices speak,\nIdeas flow, plans take shape,\nWeek's promise whispers.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Gathered voices speak,\nIdeas flow, plans take shape,\nWeek's promise whispers.", "href": null}], "page_last_edited_time": "2021-10-19T13:33:00.000Z"}, "emitted_at": 1697023326959} +{"stream": "comments", "data": {"object": "comment", "id": "e2087302-7eab-4e1d-a95b-472404fb51a2", "parent": {"type": "page_id", "page_id": "249f3796-7e81-47b0-9075-00ed2d06439d"}, "discussion_id": "be372b5d-2610-435e-981a-9be271874b8e", "created_time": "2023-09-12T20:55:00.000Z", "last_edited_time": "2023-09-12T20:55:00.000Z", "created_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "rich_text": [{"type": "text", "text": {"content": "Ladybug in flight; Red and black on petal\u2019s edge; Spring\u2019s tiny delight.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Ladybug in flight; Red and black on petal\u2019s edge; Spring\u2019s tiny delight.", "href": null}], "page_last_edited_time": "2021-10-19T13:33:00.000Z"}, "emitted_at": 1697023328874} +{"stream": "comments", "data": {"object": "comment", "id": "3f7ce236-fbc5-4b10-bb75-a0835c00aff3", "parent": {"type": "page_id", "page_id": "29299296-ef3f-4aff-aef5-02d651a59be3"}, "discussion_id": "c30808ec-fabd-4e7b-947f-a36dbec5c1db", "created_time": "2023-09-12T20:56:00.000Z", "last_edited_time": "2023-09-12T20:56:00.000Z", "created_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "rich_text": [{"type": "text", "text": {"content": "APIs converge; Fixing gaps in code and docs; Two worlds now as one.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "APIs converge; Fixing gaps in code and docs; Two worlds now as one.", "href": null}], "page_last_edited_time": "2021-10-19T13:33:00.000Z"}, "emitted_at": 1697023329304} diff --git a/source-notion/integration_tests/incremental_catalog.json b/source-notion/integration_tests/incremental_catalog.json new file mode 100644 index 0000000000..478309ad76 --- /dev/null +++ b/source-notion/integration_tests/incremental_catalog.json @@ -0,0 +1,46 @@ +{ + "streams": [ + { + "stream": { + "name": "databases", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_primary_key": [["id"]], + "default_cursor_field": ["last_edited_time"] + }, + "primary_key": [["id"]], + "cursor_field": ["last_edited_time"], + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "pages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_primary_key": [["id"]], + "default_cursor_field": ["last_edited_time"] + }, + "primary_key": [["id"]], + "cursor_field": ["last_edited_time"], + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "blocks", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_primary_key": [["id"]], + "default_cursor_field": ["last_edited_time"] + }, + "primary_key": [["id"]], + "cursor_field": ["last_edited_time"], + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/source-notion/integration_tests/invalid_config.json b/source-notion/integration_tests/invalid_config.json new file mode 100644 index 0000000000..cad13a3770 --- /dev/null +++ b/source-notion/integration_tests/invalid_config.json @@ -0,0 +1,9 @@ +{ + "start_date": "2021-01-01T00:00:00.000Z", + "credentials": { + "auth_type": "OAuth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "access_token": "access_token" + } +} diff --git a/source-notion/integration_tests/sample_config.json b/source-notion/integration_tests/sample_config.json new file mode 100644 index 0000000000..339023ce3b --- /dev/null +++ b/source-notion/integration_tests/sample_config.json @@ -0,0 +1,7 @@ +{ + "start_date": "2021-01-01T00:00:00.000Z", + "credentials": { + "auth_type": "Token", + "access_token_": "" + } +} diff --git a/source-notion/integration_tests/sample_config_oauth.json b/source-notion/integration_tests/sample_config_oauth.json new file mode 100644 index 0000000000..2aedd62665 --- /dev/null +++ b/source-notion/integration_tests/sample_config_oauth.json @@ -0,0 +1,9 @@ +{ + "start_date": "2021-01-01T00:00:00.000Z", + "credentials": { + "auth_type": "OAuth", + "client_id": "", + "client_secret": "", + "access_token": "" + } +} diff --git a/source-notion/integration_tests/sample_state.json b/source-notion/integration_tests/sample_state.json new file mode 100644 index 0000000000..0c49b5a9f4 --- /dev/null +++ b/source-notion/integration_tests/sample_state.json @@ -0,0 +1,46 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "last_edited_time": "2021-10-10T04:40:00.000Z" + }, + "stream_descriptor": { + "name": "databases" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last_edited_time": "2021-10-10T04:40:00.000Z" + }, + "stream_descriptor": { + "name": "pages" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last_edited_time": "2021-10-10T04:00:00.000Z" + }, + "stream_descriptor": { + "name": "blocks" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last_edited_time": "2021-10-10T04:00:00.000Z" + }, + "stream_descriptor": { + "name": "comments" + } + } + } +] diff --git a/source-notion/main.py b/source-notion/main.py new file mode 100644 index 0000000000..671d6cd692 --- /dev/null +++ b/source-notion/main.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_notion.run import run + +if __name__ == "__main__": + run() diff --git a/source-notion/metadata.yaml b/source-notion/metadata.yaml new file mode 100644 index 0000000000..3c6c34dd66 --- /dev/null +++ b/source-notion/metadata.yaml @@ -0,0 +1,49 @@ +data: + ab_internal: + ql: 400 + sl: 200 + allowedHosts: + hosts: + - api.notion.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + connectorSubtype: api + connectorType: source + definitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 + dockerImageTag: 2.1.0 + dockerRepository: airbyte/source-notion + documentationUrl: https://docs.airbyte.com/integrations/sources/notion + githubIssueLabel: source-notion + icon: notion.svg + license: MIT + name: Notion + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-notion + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: generally_available + releases: + breakingChanges: + 2.0.0: + message: + Version 2.0.0 introduces schema changes to multiple properties shared + by the blocks, databases and pages streams. These changes were introduced + to reflect updates to the Notion API. A full schema refresh and data reset + are required to upgrade to this version. + upgradeDeadline: "2023-11-09" + suggestedStreams: + streams: + - blocks + - databases + - pages + - users + supportLevel: certified + tags: + - language:python + - cdk:python +metadataSpecVersion: "1.0" diff --git a/source-notion/poetry.lock b/source-notion/poetry.lock new file mode 100644 index 0000000000..deaacddf79 --- /dev/null +++ b/source-notion/poetry.lock @@ -0,0 +1,1032 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.52.7" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.52.7.tar.gz", hash = "sha256:73c55464ed57e030681fb4407613a5c0f07c519a4ba63aa9a4dd43d05cdf100b"}, + {file = "airbyte_cdk-0.52.7-py3-none-any.whl", hash = "sha256:f11665fc8f8dd2632d94e57f53991f7aaba8e9643a8ee7557f3040e40ea677ff"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.19)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.19)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.19)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.19)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "setuptools" +version = "69.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9,<3.12" +content-hash = "fde07907def42fe31b6513c3d374b49cd501667cb0368ed468012b58391fb29f" diff --git a/source-notion/pyproject.toml b/source-notion/pyproject.toml new file mode 100644 index 0000000000..3c42507630 --- /dev/null +++ b/source-notion/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "2.1.0" +name = "source-notion" +description = "Source implementation for Notion." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/notion" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +[[tool.poetry.packages]] +include = "source_notion" + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "==0.52.7" +pendulum = "==2.1.2" + +[tool.poetry.scripts] +source-notion = "source_notion.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.11.0" +pytest-mock = "^3.6.1" +freezegun = "^1.4.0" +pytest = "^6.1" diff --git a/source-notion/source_notion/__init__.py b/source-notion/source_notion/__init__.py new file mode 100644 index 0000000000..6b47201e71 --- /dev/null +++ b/source-notion/source_notion/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceNotion + +__all__ = ["SourceNotion"] diff --git a/source-notion/source_notion/run.py b/source-notion/source_notion/run.py new file mode 100644 index 0000000000..df14df5ee9 --- /dev/null +++ b/source-notion/source_notion/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_notion import SourceNotion + + +def run(): + source = SourceNotion() + launch(source, sys.argv[1:]) diff --git a/source-notion/source_notion/schemas/blocks.json b/source-notion/source_notion/schemas/blocks.json new file mode 100644 index 0000000000..0e7131bfde --- /dev/null +++ b/source-notion/source_notion/schemas/blocks.json @@ -0,0 +1,289 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": ["block"] + }, + "parent": { + "$ref": "parent.json" + }, + "id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "$ref": "user.json" + }, + "last_edited_by": { + "$ref": "user.json" + }, + "last_edited_time": { + "type": "string", + "format": "date-time" + }, + "archived": { + "type": "boolean" + }, + "has_children": { + "type": ["null", "boolean"] + }, + "type": { + "enum": [ + "bookmark", + "breadcrumb", + "bulleted_list_item", + "callout", + "child_database", + "child_page", + "code", + "column", + "column_list", + "divider", + "embed", + "equation", + "file", + "heading_1", + "heading_2", + "heading_3", + "image", + "link_preview", + "link_to_page", + "numbered_list_item", + "paragraph", + "pdf", + "quote", + "synced_block", + "table", + "table_of_contents", + "table_row", + "template", + "to_do", + "toggle", + "unsupported", + "video" + ] + }, + "bookmark": { + "type": "object", + "properties": { + "url": { "type": "string" }, + "caption": { "type": "array", "items": { "$ref": "rich_text.json" } } + } + }, + "breadcrumb": { + "type": "object" + }, + "bulleted_list_item": { "$ref": "text_element.json" }, + "callout": { + "type": "object", + "properties": { + "color": { "type": "string" }, + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "icon": { "$ref": "icon.json" } + } + }, + "child_page": { "$ref": "child.json" }, + "child_database": { "$ref": "child.json" }, + "code": { + "type": "object", + "properties": { + "caption": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "language": { + "enum": [ + "abap", + "arduino", + "bash", + "basic", + "c", + "clojure", + "coffeescript", + "c++", + "c#", + "css", + "dart", + "diff", + "docker", + "elixir", + "elm", + "erlang", + "flow", + "fortran", + "f#", + "gherkin", + "glsl", + "go", + "graphql", + "groovy", + "haskell", + "html", + "java", + "javascript", + "json", + "julia", + "kotlin", + "latex", + "less", + "lisp", + "livescript", + "lua", + "makefile", + "markdown", + "markup", + "matlab", + "mermaid", + "nix", + "objective-c", + "ocaml", + "pascal", + "perl", + "php", + "plain text", + "powershell", + "prolog", + "protobuf", + "python", + "r", + "reason", + "ruby", + "rust", + "sass", + "scala", + "scheme", + "scss", + "shell", + "sql", + "swift", + "typescript", + "vb.net", + "verilog", + "vhdl", + "visual basic", + "webassembly", + "xml", + "yaml", + "java/c/c++/c#" + ] + } + } + }, + "column": { + "type": "object" + }, + "column_list": { + "type": "object" + }, + "divider": { + "type": "object" + }, + "embed": { + "type": "object", + "properties": { + "url": { "type": "string" } + } + }, + "equation": { + "type": "object", + "properties": { + "expression": { "type": "string" } + } + }, + "file": { "$ref": "file.json" }, + "heading_1": { "$ref": "heading.json" }, + "heading_2": { "$ref": "heading.json" }, + "heading_3": { "$ref": "heading.json" }, + "image": { "$ref": "file.json" }, + "link_preview": { + "type": "object", + "properties": { + "url": { "type": "string" } + } + }, + "link_to_page": { + "type": "object", + "properties": { + "page_id": { "type": "string" }, + "type": { "type": "string" } + } + }, + "numbered_list_item": { "$ref": "text_element.json" }, + "paragraph": { "$ref": "text_element.json" }, + "pdf": { "$ref": "file.json" }, + "quote": { "$ref": "text_element.json" }, + "synced_block": { + "type": "object", + "properties": { + "synced_from": { + "type": ["null", "object"], + "properties": { + "type": { + "type": "string", + "enum": ["block_id"] + }, + "block_id": { + "type": "string" + } + } + }, + "children": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + } + } + }, + "table": { + "type": "object", + "properties": { + "table_width": { "type": "integer" }, + "has_column_header": { "type": "boolean" }, + "has_row_header": { "type": "boolean" } + } + }, + "table_of_contents": { + "type": "object", + "properties": { + "color": { "type": "string" } + } + }, + "table_row": { + "type": "object", + "properties": { + "cells": { + "type": ["null", "array"], + "items": { + "type": ["null", "array"], + "items": { "$ref": "rich_text.json" } + } + } + } + }, + "template": { + "type": "object", + "properties": { + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } } + } + }, + "to_do": { + "type": "object", + "properties": { + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "checked": { "type": ["null", "boolean"] }, + "color": { "type": "string" }, + "children": { "type": "array", "items": { "type": "object" } } + } + }, + "toggle": { "$ref": "text_element.json" }, + "video": { "$ref": "file.json" }, + "unsupported": { + "type": "object" + } + } +} diff --git a/source-notion/source_notion/schemas/comments.json b/source-notion/source_notion/schemas/comments.json new file mode 100644 index 0000000000..1ab379a06e --- /dev/null +++ b/source-notion/source_notion/schemas/comments.json @@ -0,0 +1,93 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": ["comment"] + }, + "id": { + "type": "string" + }, + "parent": { + "type": "object", + "properties": { + "type": { + "enum": ["page_id"] + }, + "page_id": { + "type": "string" + } + } + }, + "discussion_id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "last_edited_time": { + "type": "string", + "format": "date-time" + }, + "page_last_edited_time": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "$ref": "user.json" + }, + "rich_text": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "text": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "link": { + "type": ["null", "object"] + } + } + }, + "annotations": { + "type": "object", + "properties": { + "bold": { + "type": "boolean" + }, + "italic": { + "type": "boolean" + }, + "strikethrough": { + "type": "boolean" + }, + "underline": { + "type": "boolean" + }, + "code": { + "type": "boolean" + }, + "color": { + "type": "string" + } + } + }, + "plain_text": { + "type": "string" + }, + "href": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/source-notion/source_notion/schemas/databases.json b/source-notion/source_notion/schemas/databases.json new file mode 100644 index 0000000000..55f004c522 --- /dev/null +++ b/source-notion/source_notion/schemas/databases.json @@ -0,0 +1,555 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": ["database"] + }, + "id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "last_edited_time": { + "type": "string", + "format": "date-time" + }, + "title": { + "type": "array", + "items": { + "$ref": "rich_text.json" + } + }, + "description": { + "type": "array", + "items": { + "$ref": "rich_text.json" + } + }, + "last_edited_by": { + "$ref": "user.json" + }, + "created_by": { + "$ref": "user.json" + }, + "archived": { + "type": "boolean" + }, + "icon": { + "$ref": "icon.json" + }, + "cover": { + "$ref": "file.json" + }, + "parent": { + "$ref": "parent.json" + }, + "url": { + "type": "string" + }, + "is_inline": { + "type": ["null", "boolean"] + }, + "public_url": { + "type": ["null", "string"] + }, + "properties": { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": "object", + "additionalProperties": true, + "anyOf": [ + { + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": [ + "checkbox", + "created_by", + "created_time", + "date", + "email", + "files", + "formula", + "last_edited_by", + "last_edited_time", + "multi_select", + "number", + "people", + "phone_number", + "relation", + "rich_text", + "rollup", + "select", + "status", + "title", + "url" + ] + }, + "name": { + "type": "string" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["checkbox"] + }, + "name": { + "type": "string" + }, + "checkbox": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["created_by"] + }, + "name": { + "type": "string" + }, + "created_by": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["created_time"] + }, + "name": { + "type": "string" + }, + "created_time": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["date"] + }, + "name": { + "type": "string" + }, + "date": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["email"] + }, + "name": { + "type": "string" + }, + "email": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["files"] + }, + "name": { + "type": "string" + }, + "files": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["formula"] + }, + "name": { + "type": "string" + }, + "expression": { + "type": "string" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["last_edited_by"] + }, + "name": { + "type": "string" + }, + "last_edited_by": { + "type": "string" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["last_edited_time"] + }, + "name": { + "type": "string" + }, + "last_edited_time": { + "type": "string" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["number"] + }, + "name": { + "type": "string" + }, + "format": { + "enum": [ + "argentine_peso", + "baht", + "canadian_dollar", + "chilean_peso", + "colombian_peso", + "danish_krone", + "dirham", + "dollar", + "euro", + "forint", + "franc", + "hong_kong_dollar", + "koruna", + "krona", + "leu", + "lira", + "mexican_peso", + "new_taiwan_dollar", + "new_zealand_dollar", + "norwegian_krone", + "number", + "number_with_commas", + "percent", + "peruvian_sol", + "philippine_peso", + "pound", + "rand", + "real", + "ringgit", + "riyal", + "ruble", + "rupee", + "rupiah", + "shekel", + "singapore_dollar", + "uruguayan_peso", + "won", + "yen", + "yuan", + "zloty" + ] + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["select", "multi_select"] + }, + "name": { + "type": "string" + }, + "options": { + "type": "array", + "items": { + "$ref": "options.json" + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["people"] + }, + "name": { + "type": "string" + }, + "people": { + "type": "string" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["phone_number"] + }, + "name": { + "type": "string" + }, + "phone_number": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["relation"] + }, + "name": { + "type": "string" + }, + "database_id": { + "type": "string" + }, + "synced_property_name": { + "type": ["null", "string"] + }, + "synced_property_id": { + "type": ["null", "string"] + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["rich_text"] + }, + "name": { + "type": "string" + }, + "rich_text": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["rollup"] + }, + "name": { + "type": "string" + }, + "relation_property_name": { + "type": "string" + }, + "relation_property_id": { + "type": "string" + }, + "rollup_property_name": { + "type": "string" + }, + "rollup_property_id": { + "type": "string" + }, + "function": { + "enum": [ + "average", + "checked", + "count_per_group", + "count", + "count_values", + "date_range", + "earliest_date", + "empty", + "latest_date", + "max", + "median", + "min", + "not_empty", + "percent_checked", + "percent_empty", + "percent_not_empty", + "percent_per_group", + "percent_unchecked", + "range", + "unchecked", + "unique", + "show_original", + "show_unique", + "sum" + ] + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["status"] + }, + "name": { + "type": "string" + }, + "status": { + "type": "object", + "properties": { + "options": { + "type": "array", + "items": { + "$ref": "options.json" + } + }, + "groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "color": { + "type": "string" + }, + "option_ids": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["title"] + }, + "name": { + "type": "string" + }, + "title": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["url"] + }, + "name": { + "type": "string" + }, + "url": { + "type": "object" + } + } + } + ] + } + } + } + } + } +} diff --git a/source-notion/source_notion/schemas/pages.json b/source-notion/source_notion/schemas/pages.json new file mode 100644 index 0000000000..7972b07d6c --- /dev/null +++ b/source-notion/source_notion/schemas/pages.json @@ -0,0 +1,330 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": ["page"] + }, + "id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "$ref": "user.json" + }, + "last_edited_time": { + "type": "string", + "format": "date-time" + }, + "last_edited_by": { + "$ref": "user.json" + }, + "archived": { + "type": "boolean" + }, + "icon": { + "$ref": "icon.json" + }, + "cover": { + "$ref": "file.json" + }, + "parent": { + "$ref": "parent.json" + }, + "url": { + "type": "string" + }, + "public_url": { + "type": ["null", "string"] + }, + "properties": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true, + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": "object", + "additionalProperties": true, + "oneOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["title"] + }, + "title": { "$ref": "title.json" } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["rich_text"] + }, + "rich_text": { + "type": ["null", "array"], + "items": { "$ref": "rich_text.json" } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["select"] }, + "select": { "$ref": "options.json" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["multi_select"] }, + "multi_select": { + "type": ["null", "array"], + "items": { "$ref": "options.json" } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["date"] }, + "date": { "$ref": "date.json" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["formula"] }, + "formula": { + "type": ["null", "object"], + "properties": { + "type": { + "enum": ["string", "number", "boolean", "date"] + }, + "string": { "type": ["null", "string"] }, + "number": { "type": ["null", "number"] }, + "boolean": { "type": ["null", "boolean"] }, + "date": { "$ref": "date.json" } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["relation"] }, + "relation": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { "type": "string" } + } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["rollup"] }, + "rollup": { + "type": ["null", "object"], + "properties": { + "type": { "enum": ["number", "date", "array"] }, + "number": { "type": ["null", "number"] }, + "date": { "$ref": "date.json" }, + "array": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "type": { "type": "string" } + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["people"] }, + "people": { + "type": ["null", "array"], + "items": { + "$ref": "user.json" + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["files"] }, + "files": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "type": { "enum": ["external", "file"] }, + "url": { "type": "string" }, + "expiry_time": { "type": ["null", "string"] }, + "name": { "type": "string" } + } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["checkbox"] }, + "checkout": { + "type": ["null", "boolean"] + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["url"] }, + "url": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["email"] }, + "email": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["phone_number"] }, + "phone_number": { "type": "object" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["created_time"] }, + "created_time": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["created_by"] }, + "created_by": { "$ref": "user.json" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["last_edited_time"] }, + "last_edited_time": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["last_edited_by"] }, + "last_edited_by": { "$ref": "user.json" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["number"] }, + "number": { + "type": "object", + "properties": { + "format": { "type": "string" } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["status"] }, + "status": { "$ref": "options.json" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["unique_id"] }, + "unique_id": { + "type": "object", + "properties": { + "number": { "type": "number" }, + "prefix": { "type": ["null", "string"] } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["verification"] + }, + "verification": { + "type": "object", + "properties": { + "state": { + "enum": ["verified", "unverified"] + }, + "verified_by": { "$ref": "user.json" }, + "date": { "$ref": "date.json" } + } + } + } + } + ] + } + } + } + } + } +} diff --git a/source-notion/source_notion/schemas/shared/child.json b/source-notion/source_notion/schemas/shared/child.json new file mode 100644 index 0000000000..dbd8ed4b14 --- /dev/null +++ b/source-notion/source_notion/schemas/shared/child.json @@ -0,0 +1,8 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "title": { "type": "string" } + } +} diff --git a/source-notion/source_notion/schemas/shared/date.json b/source-notion/source_notion/schemas/shared/date.json new file mode 100644 index 0000000000..247fb920f7 --- /dev/null +++ b/source-notion/source_notion/schemas/shared/date.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "start": { "type": ["null", "string"] }, + "end": { "type": ["null", "string"] }, + "time_zone": { "type": ["null", "string"] } + } +} diff --git a/source-notion/source_notion/schemas/shared/emoji.json b/source-notion/source_notion/schemas/shared/emoji.json new file mode 100644 index 0000000000..9d74da0c17 --- /dev/null +++ b/source-notion/source_notion/schemas/shared/emoji.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "emoji": { + "type": "string" + } + } +} diff --git a/source-notion/source_notion/schemas/shared/file.json b/source-notion/source_notion/schemas/shared/file.json new file mode 100644 index 0000000000..38dbff8d09 --- /dev/null +++ b/source-notion/source_notion/schemas/shared/file.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "type": { + "enum": ["file", "external"] + }, + "caption": { + "type": ["null", "array"], + "items": { + "$ref": "rich_text.json" + } + }, + "external": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + } + }, + "file": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + }, + "expiry_time": { + "type": "string", + "format": "date-time" + } + } + } + } +} diff --git a/source-notion/source_notion/schemas/shared/heading.json b/source-notion/source_notion/schemas/shared/heading.json new file mode 100644 index 0000000000..b321541d43 --- /dev/null +++ b/source-notion/source_notion/schemas/shared/heading.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "color": { "type": "string" }, + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "is_toggleable": { "type": "boolean" } + } +} diff --git a/source-notion/source_notion/schemas/shared/icon.json b/source-notion/source_notion/schemas/shared/icon.json new file mode 100644 index 0000000000..4ac4fe2b83 --- /dev/null +++ b/source-notion/source_notion/schemas/shared/icon.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "anyOf": [{ "$ref": "file.json" }, { "$ref": "emoji.json" }] +} diff --git a/source-notion/source_notion/schemas/shared/options.json b/source-notion/source_notion/schemas/shared/options.json new file mode 100644 index 0000000000..dcd5c95aaa --- /dev/null +++ b/source-notion/source_notion/schemas/shared/options.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "color": { + "enum": [ + "default", + "gray", + "brown", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "red" + ] + } + } +} diff --git a/source-notion/source_notion/schemas/shared/parent.json b/source-notion/source_notion/schemas/shared/parent.json new file mode 100644 index 0000000000..db2cf20b9e --- /dev/null +++ b/source-notion/source_notion/schemas/shared/parent.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "type": { + "enum": ["block_id", "database_id", "page_id", "workspace"] + }, + "block_id": { + "type": "string" + }, + "database_id": { + "type": "string" + }, + "page_id": { + "type": "string" + }, + "workspace": { + "type": "boolean" + } + } +} diff --git a/source-notion/source_notion/schemas/shared/rich_text.json b/source-notion/source_notion/schemas/shared/rich_text.json new file mode 100644 index 0000000000..e8ecf832da --- /dev/null +++ b/source-notion/source_notion/schemas/shared/rich_text.json @@ -0,0 +1,89 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": ["null", "string"] + }, + "text": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "content": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "type": { + "enum": ["url"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + }, + "mention": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "type": { + "type": ["null", "string"] + }, + "info": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + } + } + } + } + }, + "equation": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "expression": { + "type": ["null", "string"] + } + } + }, + "annotations": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "bold": { + "type": ["null", "boolean"] + }, + "italic": { + "type": ["null", "boolean"] + }, + "strikethrough": { + "type": ["null", "boolean"] + }, + "underline": { + "type": ["null", "boolean"] + }, + "code": { + "type": ["null", "boolean"] + }, + "color": { + "type": ["null", "string"] + } + } + }, + "plain_text": { + "type": ["null", "string"] + }, + "href": { + "type": ["null", "string"] + } + } +} diff --git a/source-notion/source_notion/schemas/shared/text_element.json b/source-notion/source_notion/schemas/shared/text_element.json new file mode 100644 index 0000000000..65411d825b --- /dev/null +++ b/source-notion/source_notion/schemas/shared/text_element.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "color": { "type": "string" }, + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "children": { "type": "array", "items": { "type": "object" } } + } +} diff --git a/source-notion/source_notion/schemas/shared/title.json b/source-notion/source_notion/schemas/shared/title.json new file mode 100644 index 0000000000..5e2d3c7d82 --- /dev/null +++ b/source-notion/source_notion/schemas/shared/title.json @@ -0,0 +1,62 @@ +{ + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "type": { + "type": ["null", "string"] + }, + "text": { + "type": ["null", "object"], + "properties": { + "content": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "type": { + "enum": ["url"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + }, + "annotations": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "bold": { + "type": ["null", "boolean"] + }, + "italic": { + "type": ["null", "boolean"] + }, + "strikethrough": { + "type": ["null", "boolean"] + }, + "underline": { + "type": ["null", "boolean"] + }, + "code": { + "type": ["null", "boolean"] + }, + "color": { + "type": ["null", "string"] + } + } + }, + "plain_text": { + "type": ["null", "string"] + }, + "href": { + "type": ["null", "string"] + } + } + } +} diff --git a/source-notion/source_notion/schemas/shared/user.json b/source-notion/source_notion/schemas/shared/user.json new file mode 100644 index 0000000000..8c5000dba2 --- /dev/null +++ b/source-notion/source_notion/schemas/shared/user.json @@ -0,0 +1,76 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": ["user"] + }, + "id": { + "type": "string" + }, + "name": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "type": { + "enum": ["person", "bot"] + }, + "person": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "email": { + "type": ["null", "string"] + } + } + }, + "bot": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "owner": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "info": { + "avatar_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "person": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + "workspace": { + "type": ["null", "boolean"] + } + } + }, + "workspace_name": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/source-notion/source_notion/schemas/users.json b/source-notion/source_notion/schemas/users.json new file mode 100644 index 0000000000..b63194f13f --- /dev/null +++ b/source-notion/source_notion/schemas/users.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "user.json" +} diff --git a/source-notion/source_notion/source.py b/source-notion/source_notion/source.py new file mode 100644 index 0000000000..08262edfee --- /dev/null +++ b/source-notion/source_notion/source.py @@ -0,0 +1,98 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +import re +from typing import Any, List, Mapping, Tuple + +import pendulum +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from pendulum.parsing.exceptions import ParserError + +from .streams import Blocks, Comments, Databases, Pages, Users + + +class SourceNotion(AbstractSource): + def _get_authenticator(self, config: Mapping[str, Any]) -> TokenAuthenticator: + credentials = config.get("credentials", {}) + auth_type = credentials.get("auth_type") + token = credentials.get("access_token") if auth_type == "OAuth2.0" else credentials.get("token") + + if credentials and token: + return TokenAuthenticator(token) + + # The original implementation did not support OAuth, and therefore had no "credentials" key. + # We can maintain backwards compatibility for OG connections by checking for the deprecated "access_token" key, just in case. + if config.get("access_token"): + return TokenAuthenticator(config["access_token"]) + + def _validate_start_date(self, config: Mapping[str, Any]): + start_date = config.get("start_date") + + if start_date: + pattern = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z") + if not pattern.match(start_date): # Compare against the pattern descriptor. + return "Please check the format of the start date against the pattern descriptor." + + try: # Handle invalid dates. + parsed_start_date = pendulum.parse(start_date) + except ParserError: + return "The provided start date is not a valid date. Please check the format and try again." + + if parsed_start_date > pendulum.now("UTC"): # Handle future start date. + return "The start date cannot be greater than the current date." + + return None + + def _extract_error_message(self, response: requests.Response) -> str: + """ + Return a human-readable error message from a Notion API response, for use in connection check. + """ + error_json = response.json() + error_code = error_json.get("code", "unknown_error") + error_message = error_json.get( + "message", "An unspecified error occurred while connecting to Notion. Please check your credentials and try again." + ) + + if error_code == "unauthorized": + return "The provided API access token is invalid. Please double-check that you input the correct token and have granted the necessary permissions to your Notion integration." + if error_code == "restricted_resource": + return "The provided API access token does not have the correct permissions configured. Please double-check that you have granted all the necessary permissions to your Notion integration." + return f"Error: {error_message} (Error code: {error_code})" + + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]: + # First confirm that if start_date is set by user, it is valid. + validation_error = self._validate_start_date(config) + if validation_error: + return False, validation_error + try: + authenticator = self._get_authenticator(config) + # Notion doesn't have a dedicated ping endpoint, so we can use the users/me endpoint instead. + # Endpoint docs: https://developers.notion.com/reference/get-self + ping_endpoint = "https://api.notion.com/v1/users/me" + notion_version = {"Notion-Version": "2022-06-28"} + response = requests.get(ping_endpoint, auth=authenticator, headers=notion_version) + + if response.status_code == 200: + return True, None + else: + error_message = self._extract_error_message(response) + return False, error_message + + except requests.exceptions.RequestException as e: + return False, str(e) + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + + authenticator = self._get_authenticator(config) + args = {"authenticator": authenticator, "config": config} + pages = Pages(**args) + blocks = Blocks(parent=pages, **args) + comments = Comments(parent=pages, **args) + + return [Users(**args), Databases(**args), pages, blocks, comments] diff --git a/source-notion/source_notion/spec.json b/source-notion/source_notion/spec.json new file mode 100644 index 0000000000..4b833a5674 --- /dev/null +++ b/source-notion/source_notion/spec.json @@ -0,0 +1,119 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/notion", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Notion Source Spec", + "type": "object", + "required": ["credentials"], + "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format YYYY-MM-DDTHH:MM:SS.000Z. During incremental sync, any data generated before this date will not be replicated. If left blank, the start date will be set to 2 years before the present date.", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:MM:SS.000Z", + "examples": ["2020-11-16T00:00:00.000Z"], + "type": "string", + "format": "date-time" + }, + "credentials": { + "title": "Authentication Method", + "description": "Choose either OAuth (recommended for Airbyte Cloud) or Access Token. See our docs for more information.", + "type": "object", + "order": 1, + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": [ + "auth_type", + "client_id", + "client_secret", + "access_token" + ], + "properties": { + "auth_type": { + "type": "string", + "const": "OAuth2.0" + }, + "client_id": { + "title": "Client ID", + "type": "string", + "description": "The Client ID of your Notion integration. See our docs for more information.", + "airbyte_secret": true + }, + "client_secret": { + "title": "Client Secret", + "type": "string", + "description": "The Client Secret of your Notion integration. See our docs for more information.", + "airbyte_secret": true + }, + "access_token": { + "title": "Access Token", + "type": "string", + "description": "The Access Token received by completing the OAuth flow for your Notion integration. See our docs for more information.", + "airbyte_secret": true + } + } + }, + { + "type": "object", + "title": "Access Token", + "required": ["auth_type", "token"], + "properties": { + "auth_type": { + "type": "string", + "const": "token" + }, + "token": { + "title": "Access Token", + "description": "The Access Token for your private Notion integration. See the docs for more information on how to obtain this token.", + "type": "string", + "airbyte_secret": true + } + } + } + ] + } + } + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "OAuth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } + } +} diff --git a/source-notion/source_notion/streams.py b/source-notion/source_notion/streams.py new file mode 100644 index 0000000000..f7cb9e456e --- /dev/null +++ b/source-notion/source_notion/streams.py @@ -0,0 +1,453 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, TypeVar + +import pendulum +import pydantic +import requests +from airbyte_cdk.logger import AirbyteLogger as Logger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy +from airbyte_cdk.sources.streams.http.exceptions import UserDefinedBackoffException +from requests import HTTPError + +from .utils import transform_properties + +# maximum block hierarchy recursive request depth +MAX_BLOCK_DEPTH = 30 + + +class NotionAvailabilityStrategy(HttpAvailabilityStrategy): + """ + Inherit from HttpAvailabilityStrategy with slight modification to 403 error message. + """ + + def reasons_for_unavailable_status_codes(self, stream: Stream, logger: Logger, source: Source, error: HTTPError) -> Dict[int, str]: + + reasons_for_codes: Dict[int, str] = { + requests.codes.FORBIDDEN: "This is likely due to insufficient permissions for your Notion integration. " + "Please make sure your integration has read access for the resources you are trying to sync" + } + return reasons_for_codes + + +class NotionStream(HttpStream, ABC): + + url_base = "https://api.notion.com/v1/" + + primary_key = "id" + + page_size = 100 # set by Notion API spec + + raise_on_http_errors = True + + def __init__(self, config: Mapping[str, Any], **kwargs): + super().__init__(**kwargs) + self.start_date = config.get("start_date") + + # If start_date is not found in config, set it to 2 years ago and update value in config for use in next stream + if not self.start_date: + self.start_date = pendulum.now().subtract(years=2).in_timezone("UTC").format("YYYY-MM-DDTHH:mm:ss.SSS[Z]") + config["start_date"] = self.start_date + + @property + def availability_strategy(self) -> HttpAvailabilityStrategy: + return NotionAvailabilityStrategy() + + @property + def retry_factor(self) -> int: + return 5 + + @property + def max_retries(self) -> int: + return 7 + + @property + def max_time(self) -> int: + return 60 * 11 + + @staticmethod + def check_invalid_start_cursor(response: requests.Response): + if response.status_code == 400: + message = response.json().get("message", "") + if message.startswith("The start_cursor provided is invalid: "): + return message + + @staticmethod + def throttle_request_page_size(current_page_size): + """ + Helper method to halve page_size when encountering a 504 Gateway Timeout error. + """ + throttled_page_size = max(current_page_size // 2, 10) + return throttled_page_size + + def backoff_time(self, response: requests.Response) -> Optional[float]: + """ + Notion's rate limit is approx. 3 requests per second, with larger bursts allowed. + For a 429 response, we can use the retry-header to determine how long to wait before retrying. + For 500-level errors, we use Airbyte CDK's default exponential backoff with a retry_factor of 5. + Docs: https://developers.notion.com/reference/errors#rate-limiting + """ + retry_after = response.headers.get("retry-after", "5") + if response.status_code == 429: + return float(retry_after) + if self.check_invalid_start_cursor(response): + return 10 + return super().backoff_time(response) + + def should_retry(self, response: requests.Response) -> bool: + # In the case of a 504 Gateway Timeout error, we can lower the page_size when retrying to reduce the load on the server. + if response.status_code == 504: + self.page_size = self.throttle_request_page_size(self.page_size) + self.logger.info(f"Encountered a server timeout. Reducing request page size to {self.page_size} and retrying.") + + # If page_size has been reduced after encountering a 504 Gateway Timeout error, + # we increase it back to the default of 100 once a success response is achieved, for the following API calls. + if response.status_code == 200 and self.page_size != 100: + self.page_size = 100 + self.logger.info(f"Successfully reconnected after a server timeout. Increasing request page size to {self.page_size}.") + + return response.status_code == 400 or super().should_retry(response) + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + params = super().request_headers(**kwargs) + # Notion API version, see https://developers.notion.com/reference/versioning + params["Notion-Version"] = "2022-06-28" + return params + + def next_page_token( + self, + response: requests.Response, + ) -> Optional[Mapping[str, Any]]: + """ + An example of response: + { + "next_cursor": "fe2cc560-036c-44cd-90e8-294d5a74cebc", + "has_more": true, + "results": [ ... ] + } + Doc: https://developers.notion.com/reference/intro#pagination + """ + next_cursor = response.json().get("next_cursor") + if next_cursor: + return {"next_cursor": next_cursor} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + # sometimes notion api returns response without results object + data = response.json().get("results", []) + yield from data + + +T = TypeVar("T") + + +class StateValueWrapper(pydantic.BaseModel): + + stream: T + state_value: str + max_cursor_time = "" + + def __repr__(self): + """Overrides print view""" + return self.value + + @property + def value(self) -> str: + """Return max cursor time after stream sync is finished.""" + return self.max_cursor_time if self.stream.is_finished else self.state_value + + def dict(self, **kwargs): + """Overrides default logic to return current value only.""" + return {pydantic.utils.ROOT_KEY: self.value} + + +class IncrementalNotionStream(NotionStream, ABC): + + cursor_field = "last_edited_time" + + http_method = "POST" + + # whether the whole stream sync is finished + is_finished = True + + def __init__(self, obj_type: Optional[str] = None, **kwargs): + super().__init__(**kwargs) + + # object type for search filtering, either "page" or "database" if not None + self.obj_type = obj_type + + def path(self, **kwargs) -> str: + return "search" + + def request_body_json(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> Optional[Mapping]: + if not self.obj_type: + return + + # search request body + # Docs: https://developers.notion.com/reference/post-search + body = { + "sort": {"direction": "ascending", "timestamp": "last_edited_time"}, + "filter": {"property": "object", "value": self.obj_type}, + "page_size": self.page_size, + } + if next_page_token: + body["start_cursor"] = next_page_token["next_cursor"] + + return body + + def read_records(self, sync_mode: SyncMode, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + if sync_mode == SyncMode.full_refresh: + stream_state = None + try: + yield from super().read_records(sync_mode, stream_state=stream_state, **kwargs) + except UserDefinedBackoffException as e: + message = self.check_invalid_start_cursor(e.response) + if message: + self.logger.error(f"Skipping stream {self.name}, error message: {message}") + return + raise e + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + records = super().parse_response(response, stream_state=stream_state, **kwargs) + for record in records: + record_lmd = record.get(self.cursor_field, "") + state_lmd = stream_state.get(self.cursor_field, "") + if isinstance(state_lmd, StateValueWrapper): + state_lmd = state_lmd.value + if (not stream_state or record_lmd >= state_lmd) and record_lmd >= self.start_date: + yield from transform_properties(record) + + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any], + ) -> Mapping[str, Any]: + state_value = (current_stream_state or {}).get(self.cursor_field, "") + if not isinstance(state_value, StateValueWrapper): + state_value = StateValueWrapper(stream=self, state_value=state_value) + + record_time = latest_record.get(self.cursor_field, self.start_date) + state_value.max_cursor_time = max(state_value.max_cursor_time, record_time) + + return {self.cursor_field: state_value} + + +class Users(NotionStream): + """ + Docs: https://developers.notion.com/reference/get-users + """ + + def path(self, **kwargs) -> str: + return "users" + + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = {"page_size": self.page_size} + if next_page_token: + params["start_cursor"] = next_page_token["next_cursor"] + return params + + def transform(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + owner = record.get("bot", {}).get("owner") + if owner: + owner_type = owner.get("type") + owner_info = owner.get(owner_type) + if owner_type and owner_info: + record["bot"]["owner"]["info"] = owner_info + del record["bot"]["owner"][owner_type] + return record + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + # sometimes notion api returns response without results object + data = response.json().get("results", []) + for record in data: + yield self.transform(record) + + +class Databases(IncrementalNotionStream): + """ + Docs: https://developers.notion.com/reference/post-search + """ + + state_checkpoint_interval = 100 + + def __init__(self, **kwargs): + super().__init__(obj_type="database", **kwargs) + + +class Pages(IncrementalNotionStream): + """ + Docs: https://developers.notion.com/reference/post-search + """ + + state_checkpoint_interval = 100 + + def __init__(self, **kwargs): + super().__init__(obj_type="page", **kwargs) + + +class Blocks(HttpSubStream, IncrementalNotionStream): + """ + Docs: https://developers.notion.com/reference/get-block-children + """ + + http_method = "GET" + + # block id stack for block hierarchy traversal + block_id_stack = [] + + def path(self, **kwargs) -> str: + return f"blocks/{self.block_id_stack[-1]}/children" + + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = {"page_size": self.page_size} + if next_page_token: + params["start_cursor"] = next_page_token["next_cursor"] + return params + + def stream_slices( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Optional[Mapping[str, Any]]]: + # gather parent stream records in full + slices = list(super().stream_slices(SyncMode.full_refresh, cursor_field, stream_state)) + + self.is_finished = False + for page in slices: + page_id = page["parent"]["id"] + self.block_id_stack.append(page_id) + + # stream sync is finished when it is on the last slice + self.is_finished = page_id == slices[-1]["parent"]["id"] + + yield {"page_id": page_id} + + def transform(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + transform_object_field = record.get("type") + + if transform_object_field: + rich_text = record.get(transform_object_field, {}).get("rich_text", []) + for r in rich_text: + mention = r.get("mention") + if mention: + type_info = mention[mention["type"]] + record[transform_object_field]["rich_text"][rich_text.index(r)]["mention"]["info"] = type_info + del record[transform_object_field]["rich_text"][rich_text.index(r)]["mention"][mention["type"]] + + return record + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + # pages and databases blocks are already fetched in their streams, so no + # need to do it again + # fetching of `ai_block` type is unsupported by API + # https://github.com/airbytehq/oncall/issues/1927 + records = super().parse_response(response, stream_state=stream_state, **kwargs) + for record in records: + if record["type"] not in ("child_page", "child_database", "ai_block"): + yield self.transform(record) + + def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + # if reached recursive limit, don't read anymore + if len(self.block_id_stack) > MAX_BLOCK_DEPTH: + return + + records = super().read_records(**kwargs) + for record in records: + if record.get("has_children", False): + # do the depth first traversal recursive call, get children blocks + self.block_id_stack.append(record["id"]) + yield from self.read_records(**kwargs) + yield record + + self.block_id_stack.pop() + + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == 404: + setattr(self, "raise_on_http_errors", False) + self.logger.error( + f"Stream {self.name}: {response.json().get('message')}. 404 HTTP response returns if the block specified by id doesn't" + " exist, or if the integration doesn't have access to the block." + "See more in docs: https://developers.notion.com/reference/get-block-children" + ) + return False + + if response.status_code == 400: + error_code = response.json().get("code") + error_msg = response.json().get("message") + if "validation_error" in error_code and "ai_block is not supported" in error_msg: + setattr(self, "raise_on_http_errors", False) + self.logger.error( + f"Stream {self.name}: `ai_block` type is not supported, skipping. See https://developers.notion.com/reference/block for available block type." + ) + return False + else: + return super().should_retry(response) + return super().should_retry(response) + + +class Comments(HttpSubStream, IncrementalNotionStream): + """ + Comments Object Docs: https://developers.notion.com/reference/comment-object + Comments Endpoint Docs: https://developers.notion.com/reference/retrieve-a-comment + """ + + http_method = "GET" + # We can use the "last edited time" of the parent Page as the cursor field, + # since we cannot guarantee the order of comments between pages. + cursor_field = "page_last_edited_time" + + def path(self, **kwargs) -> str: + return "comments" + + def request_params( + self, next_page_token: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + block_id = stream_slice.get("block_id") + params = {"block_id": block_id, "page_size": self.page_size} + + if next_page_token: + params["start_cursor"] = next_page_token["next_cursor"] + + return params + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping]: + + # Get the parent's "last edited time" to compare against state + page_last_edited_time = stream_slice.get("page_last_edited_time", "") + records = response.json().get("results", []) + + for record in records: + record["page_last_edited_time"] = page_last_edited_time + state_last_edited_time = stream_state.get(self.cursor_field, "") + + if isinstance(state_last_edited_time, StateValueWrapper): + state_last_edited_time = state_last_edited_time.value + + if not stream_state or page_last_edited_time >= state_last_edited_time: + yield from transform_properties(record) + + def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + + yield from IncrementalNotionStream.read_records(self, **kwargs) + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, **kwargs + ) -> Iterable[Optional[Mapping[str, Any]]]: + + # Gather parent stream records in full + parent_records = self.parent.read_records(sync_mode=SyncMode.full_refresh, cursor_field=self.parent.cursor_field) + + # The parent stream is the Pages stream, but we have to pass its id to the request_params as "block_id" + # because pages are also blocks in the Notion API. + # We also grab the last_edited_time from the parent record to use as the cursor field. + for record in parent_records: + yield {"block_id": record["id"], "page_last_edited_time": record["last_edited_time"]} diff --git a/source-notion/source_notion/utils.py b/source-notion/source_notion/utils.py new file mode 100644 index 0000000000..6103f00fb6 --- /dev/null +++ b/source-notion/source_notion/utils.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Mapping + + +def transform_properties(record: Mapping[str, Any], dict_key: str = "properties") -> Mapping[str, Any]: + """ + Transform nested `properties` object. + Move unique named entities into `name`, `value` to handle normalization. + EXAMPLE INPUT: + { + {...}, + "properties": { + "some_unique_name1": { + "id": "some_id", + "type": "relation", + "relation": [] + }, + "some_unique_name2": { + "id": "some_id", + "type": "date", + "date": null + }, + ... + }, + {...} + } + + EXAMPLE OUTPUT: + { + {...}, + "properties": [ + { + "name": "some_unique_name1", + "value": { + "id": "some_id", + "type": "relation", + "relation": [] + } + }, + { + "name": "some_unique_name2", + "value": { + "id": "some_id", + "type": "date", + "date": null + } + }, + ], + {...} + } + + """ + properties = record.get(dict_key) + if properties: + new_properties = [] + for k, v in properties.items(): + new_properties.append({"name": k, "value": v}) + record[dict_key] = new_properties + yield record diff --git a/source-notion/unit_tests/__init__.py b/source-notion/unit_tests/__init__.py new file mode 100644 index 0000000000..46b7376756 --- /dev/null +++ b/source-notion/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/source-notion/unit_tests/test_incremental_streams.py b/source-notion/unit_tests/test_incremental_streams.py new file mode 100644 index 0000000000..d34afd131f --- /dev/null +++ b/source-notion/unit_tests/test_incremental_streams.py @@ -0,0 +1,403 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import re +import time +from unittest.mock import MagicMock, patch + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException +from pytest import fixture, mark +from source_notion.streams import Blocks, Comments, IncrementalNotionStream, Pages + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalNotionStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalNotionStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalNotionStream, "__abstractmethods__", set()) + + +@fixture +def args(): + return {"authenticator": None, "config": {"access_token": "", "start_date": "2021-01-01T00:00:00.000Z"}} + + +@fixture +def parent(args): + return Pages(**args) + + +@fixture +def stream(patch_incremental_base_class, args): + return IncrementalNotionStream(**args) + + +@fixture +def blocks(parent, args): + return Blocks(parent=parent, **args) + + +@fixture +def comments(parent, args): + return Comments(parent=parent, **args) + + +def test_cursor_field(stream): + expected_cursor_field = "last_edited_time" + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(stream): + stream.is_finished = False + + inputs = { + "current_stream_state": {"last_edited_time": "2021-10-10T00:00:00.000Z"}, + "latest_record": {"last_edited_time": "2021-10-20T00:00:00.000Z"}, + } + expected_state = "2021-10-10T00:00:00.000Z" + state = stream.get_updated_state(**inputs) + assert state["last_edited_time"].value == expected_state + + inputs = {"current_stream_state": state, "latest_record": {"last_edited_time": "2021-10-30T00:00:00.000Z"}} + state = stream.get_updated_state(**inputs) + assert state["last_edited_time"].value == expected_state + + # after stream sync is finished, state should output the max cursor time + stream.is_finished = True + inputs = {"current_stream_state": state, "latest_record": {"last_edited_time": "2021-10-10T00:00:00.000Z"}} + expected_state = "2021-10-30T00:00:00.000Z" + state = stream.get_updated_state(**inputs) + assert state["last_edited_time"].value == expected_state + + +def test_stream_slices(blocks, requests_mock): + stream = blocks + requests_mock.post( + "https://api.notion.com/v1/search", + json={ + "results": [ + {"id": "aaa", "last_edited_time": "2022-10-10T00:00:00.000Z"}, + {"id": "bbb", "last_edited_time": "2022-10-10T00:00:00.000Z"}, + ], + "next_cursor": None, + }, + ) + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + expected_stream_slice = [{"page_id": "aaa"}, {"page_id": "bbb"}] + assert list(stream.stream_slices(**inputs)) == expected_stream_slice + + +def test_end_of_stream_state(blocks, requests_mock): + stream = blocks + requests_mock.post( + "https://api.notion.com/v1/search", json={"results": [{"id": "aaa"}, {"id": "bbb"}, {"id": "ccc"}], "next_cursor": None} + ) + requests_mock.get( + "https://api.notion.com/v1/blocks/aaa/children", + json={ + "results": [{"id": "block 1", "type": "heading_1", "has_children": False, "last_edited_time": "2021-10-30T00:00:00.000Z"}], + "next_cursor": None, + }, + ) + requests_mock.get( + "https://api.notion.com/v1/blocks/bbb/children", + json={ + "results": [{"id": "block 2", "type": "heading_1", "has_children": False, "last_edited_time": "2021-10-20T00:00:00.000Z"}], + "next_cursor": None, + }, + ) + requests_mock.get( + "https://api.notion.com/v1/blocks/ccc/children", + json={ + "results": [{"id": "block 3", "type": "heading_1", "has_children": False, "last_edited_time": "2021-10-10T00:00:00.000Z"}], + "next_cursor": None, + }, + ) + + state = {"last_edited_time": "2021-10-01T00:00:00.000Z"} + sync_mode = SyncMode.incremental + + for idx, app_slice in enumerate(stream.stream_slices(sync_mode, **MagicMock())): + for record in stream.read_records(sync_mode=sync_mode, stream_slice=app_slice): + state = stream.get_updated_state(state, record) + state_value = state["last_edited_time"].value + if idx == 2: # the last slice + assert state_value == "2021-10-30T00:00:00.000Z" + else: + assert state_value == "2021-10-01T00:00:00.000Z" + + +def test_supports_incremental(stream, mocker): + mocker.patch.object(IncrementalNotionStream, "cursor_field", "dummy_field") + assert stream.supports_incremental + + +def test_source_defined_cursor(stream): + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(stream): + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval + + +def test_request_params(blocks): + stream = blocks + inputs = {"stream_state": {}, "next_page_token": {"next_cursor": "aaa"}} + expected_request_params = {"page_size": 100, "start_cursor": "aaa"} + assert stream.request_params(**inputs) == expected_request_params + + +def test_record_filter(blocks, requests_mock): + stream = blocks + sync_mode = SyncMode.incremental + + root = "aaa" + record = {"id": "id1", "type": "heading_1", "has_children": False, "last_edited_time": "2021-10-20T00:00:00.000Z"} + requests_mock.get(f"https://api.notion.com/v1/blocks/{root}/children", json={"results": [record], "next_cursor": None}) + + inputs = { + "sync_mode": sync_mode, + "stream_state": {"last_edited_time": "2021-10-10T00:00:00.000Z"}, + } + stream.block_id_stack = [root] + assert next(stream.read_records(**inputs)) == record + + inputs = { + "sync_mode": sync_mode, + "stream_state": {"last_edited_time": "2021-10-30T00:00:00.000Z"}, + } + stream.block_id_stack = [root] + assert list(stream.read_records(**inputs)) == [] + + # 'child_page' and 'child_database' should not be included + record["type"] = "child_page" + inputs = { + "sync_mode": sync_mode, + "stream_state": {"last_edited_time": "2021-10-10T00:00:00.000Z"}, + } + stream.block_id_stack = [root] + assert list(stream.read_records(**inputs)) == [] + record["type"] = "child_database" + stream.block_id_stack = [root] + assert list(stream.read_records(**inputs)) == [] + + +def test_recursive_read(blocks, requests_mock): + stream = blocks + + # block records tree: + # + # root |-> record1 -> record2 -> record3 + # |-> record4 + + root = "aaa" + record1 = {"id": "id1", "type": "heading_1", "has_children": True, "last_edited_time": "2022-10-10T00:00:00.000Z"} + record2 = {"id": "id2", "type": "heading_1", "has_children": True, "last_edited_time": "2022-10-10T00:00:00.000Z"} + record3 = {"id": "id3", "type": "heading_1", "has_children": False, "last_edited_time": "2022-10-10T00:00:00.000Z"} + record4 = {"id": "id4", "type": "heading_1", "has_children": False, "last_edited_time": "2022-10-10T00:00:00.000Z"} + requests_mock.get(f"https://api.notion.com/v1/blocks/{root}/children", json={"results": [record1, record4], "next_cursor": None}) + requests_mock.get(f"https://api.notion.com/v1/blocks/{record1['id']}/children", json={"results": [record2], "next_cursor": None}) + requests_mock.get(f"https://api.notion.com/v1/blocks/{record2['id']}/children", json={"results": [record3], "next_cursor": None}) + + inputs = {"sync_mode": SyncMode.incremental} + stream.block_id_stack = [root] + assert list(stream.read_records(**inputs)) == [record3, record2, record1, record4] + + +def test_invalid_start_cursor(parent, requests_mock, caplog): + stream = parent + error_message = "The start_cursor provided is invalid: wrong_start_cursor" + search_endpoint = requests_mock.post( + "https://api.notion.com/v1/search", + status_code=400, + json={"object": "error", "status": 400, "code": "validation_error", "message": error_message}, + ) + + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + with patch.object(stream, "backoff_time", return_value=0.1): + list(stream.read_records(**inputs)) + assert search_endpoint.call_count == 8 + assert f"Skipping stream pages, error message: {error_message}" in caplog.messages + + +@mark.parametrize( + "status_code,error_code,error_message, expected_backoff_time", + [ + (400, "validation_error", "The start_cursor provided is invalid: wrong_start_cursor", [10, 10, 10, 10, 10, 10, 10]), + (429, "rate_limited", "Rate Limited", [5, 5, 5, 5, 5, 5, 5]), # Retry-header is set to 5 seconds for test + (500, "internal_server_error", "Internal server error", [5, 10, 20, 40, 80, 5, 10]), + ], +) +def test_retry_logic(status_code, error_code, error_message, expected_backoff_time, parent, requests_mock, caplog): + stream = parent + + # Set up a generator that alternates between error and success responses, to check the reset of backoff time between failures + mock_responses = ( + [ + { + "status_code": status_code, + "response": {"object": "error", "status": status_code, "code": error_code, "message": error_message}, + } + for _ in range(5) + ] + + [{"status_code": 200, "response": {"object": "list", "results": [], "has_more": True, "next_cursor": "dummy_cursor"}}] + + [ + { + "status_code": status_code, + "response": {"object": "error", "status": status_code, "code": error_code, "message": error_message}, + } + for _ in range(2) + ] + + [{"status_code": 200, "response": {"object": "list", "results": [], "has_more": False, "next_cursor": None}}] + ) + + def response_callback(request, context): + # Get the next response from the mock_responses list + response = mock_responses.pop(0) + context.status_code = response["status_code"] + return response["response"] + + # Mock the time.sleep function to avoid waiting during tests + with patch.object(time, "sleep", return_value=None): + search_endpoint = requests_mock.post( + "https://api.notion.com/v1/search", + json=response_callback, + headers={"retry-after": "5"}, + ) + + inputs = {"sync_mode": SyncMode.full_refresh, "cursor_field": [], "stream_state": {}} + try: + list(stream.read_records(**inputs)) + except (UserDefinedBackoffException, DefaultBackoffException) as e: + return e + + # Check that the endpoint was called the expected number of times + assert search_endpoint.call_count == 9 + + # Additional assertions to check reset of backoff time + # Find the backoff times from the message logs to compare against expected backoff times + log_messages = [record.message for record in caplog.records] + backoff_times = [ + round(float(re.search(r"(\d+(\.\d+)?) seconds", msg).group(1))) + for msg in log_messages + if any(word in msg for word in ["Sleeping", "Waiting"]) + ] + + assert backoff_times == expected_backoff_time, f"Unexpected backoff times: {backoff_times}" + + +# Tests for Comments stream +def test_comments_path(comments): + assert comments.path() == "comments" + + +def test_comments_request_params(comments): + """ + Test that the request_params function returns the correct parameters for the Comments endpoint + """ + params = comments.request_params( + next_page_token=None, stream_slice={"block_id": "block1", "page_last_edited_time": "2021-01-01T00:00:00.000Z"} + ) + + assert params == {"block_id": "block1", "page_size": comments.page_size} + + +def test_comments_stream_slices(comments, requests_mock): + """ + Test that the stream_slices function returns the parent page ids as "block_id" and the last edited time as "page_last_edited_time" + """ + + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": comments.cursor_field, "stream_state": {}} + + requests_mock.post( + "https://api.notion.com/v1/search", + json={ + "results": [ + {"name": "page_1", "id": "id_1", "last_edited_time": "2021-01-01T00:00:00.000Z"}, + {"name": "page_2", "id": "id_2", "last_edited_time": "2021-20-01T00:00:00.000Z"}, + ], + "next_cursor": None, + }, + ) + + expected_stream_slice = [ + {"block_id": "id_1", "page_last_edited_time": "2021-01-01T00:00:00.000Z"}, + {"block_id": "id_2", "page_last_edited_time": "2021-20-01T00:00:00.000Z"}, + ] + + actual_stream_slices_list = list(comments.stream_slices(**inputs)) + assert actual_stream_slices_list == expected_stream_slice + + +@mark.parametrize( + "stream_slice, stream_state, mock_data, expected_records", + [ + # Test that comments with page_last_edited_time >= stream_state are replicated, regardless of each record's LMD + ( + {"block_id": "block_id_1", "page_last_edited_time": "2023-10-10T00:00:00.000Z"}, + {"page_last_edited_time": "2021-10-10T00:00:00.000Z"}, + [ + { + "id": "comment_id_1", + "rich_text": [{"type": "text", "text": {"content": "I am the Alpha comment"}}], + "last_edited_time": "2021-01-01T00:00:00.000Z", + }, + { + "id": "comment_id_2", + "rich_text": [{"type": "text", "text": {"content": "I am the Omega comment"}}], + "last_edited_time": "2022-12-31T00:00:00.000Z", + }, + ], + [ + { + "id": "comment_id_1", + "rich_text": [{"type": "text", "text": {"content": "I am the Alpha comment"}}], + "last_edited_time": "2021-01-01T00:00:00.000Z", + "page_last_edited_time": "2023-10-10T00:00:00.000Z", + }, + { + "id": "comment_id_2", + "rich_text": [{"type": "text", "text": {"content": "I am the Omega comment"}}], + "last_edited_time": "2022-12-31T00:00:00.000Z", + "page_last_edited_time": "2023-10-10T00:00:00.000Z", + }, + ], + ), + # Test that comments with page_last_edited_time < stream_state are not replicated, regardless of each record's LMD + ( + {"block_id": "block_id_2", "page_last_edited_time": "2021-01-01T00:00:00.000Z"}, + {"page_last_edited_time": "2022-20-20T00:00:00.000Z"}, + [ + { + "id": "comment_id_1", + "rich_text": [{"type": "text", "text": {"content": "I will not be replicated"}}], + "last_edited_time": "2021-10-30T00:00:00.000Z", + }, + { + "id": "comment_id_2", + "rich_text": [{"type": "text", "text": {"content": "I will also not be replicated"}}], + "last_edited_time": "2023-01-01T00:00:00.000Z", + }, + ], + [], + ), + ], +) +def test_comments_read_records(comments, requests_mock, stream_slice, stream_state, mock_data, expected_records): + inputs = { + "sync_mode": SyncMode.incremental, + "cursor_field": comments.cursor_field, + "stream_state": stream_state, + "stream_slice": stream_slice, + } + + requests_mock.get( + f"https://api.notion.com/v1/comments?block_id={stream_slice['block_id']}", json={"results": mock_data, "next_cursor": None} + ) + + response = list(comments.read_records(**inputs)) + assert response == expected_records diff --git a/source-notion/unit_tests/test_source.py b/source-notion/unit_tests/test_source.py new file mode 100644 index 0000000000..2831b1b0f8 --- /dev/null +++ b/source-notion/unit_tests/test_source.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from source_notion.source import SourceNotion + +UNAUTHORIZED_ERROR_MESSAGE = "The provided API access token is invalid. Please double-check that you input the correct token and have granted the necessary permissions to your Notion integration." +RESTRICTED_RESOURCE_ERROR_MESSAGE = "The provided API access token does not have the correct permissions configured. Please double-check that you have granted all the necessary permissions to your Notion integration." +GENERIC_ERROR_MESSAGE = "Conflict occurred while saving. Please try again." +DEFAULT_ERROR_MESSAGE = "An unspecified error occurred while connecting to Notion. Please check your credentials and try again." + + +def test_check_connection(mocker, requests_mock): + source = SourceNotion() + logger_mock, config_mock = MagicMock(), {"access_token": "test_token", "start_date": "2021-01-01T00:00:00.000Z"} + requests_mock.get( + "https://api.notion.com/v1/users/me", + json={"results": [{"id": "aaa", "last_edited_time": "2022-01-01T00:00:00.000Z"}], "next_cursor": None}, + ) + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +@pytest.mark.parametrize( + "status_code,json_response,expected_message", + [ + (401, {"code": "unauthorized"}, UNAUTHORIZED_ERROR_MESSAGE), + (403, {"code": "restricted_resource"}, RESTRICTED_RESOURCE_ERROR_MESSAGE), + (409, {"code": "conflict_error", "message": GENERIC_ERROR_MESSAGE}, f"Error: {GENERIC_ERROR_MESSAGE} (Error code: conflict_error)"), + (400, {}, f"Error: {DEFAULT_ERROR_MESSAGE} (Error code: unknown_error)"), + ], +) +def test_check_connection_errors(mocker, requests_mock, status_code, json_response, expected_message): + source = SourceNotion() + logger_mock, config_mock = MagicMock(), {"access_token": "test_token", "start_date": "2021-01-01T00:00:00.000Z"} + requests_mock.get("https://api.notion.com/v1/users/me", status_code=status_code, json=json_response) + result, message = source.check_connection(logger_mock, config_mock) + + assert result is False + assert message == expected_message + + +def test_streams(mocker): + source = SourceNotion() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 5 + assert len(streams) == expected_streams_number + + +@pytest.mark.parametrize( + "config, expected_token", + [ + ({"credentials": {"auth_type": "OAuth2.0", "access_token": "oauth_token"}}, "Bearer oauth_token"), + ({"credentials": {"auth_type": "token", "token": "other_token"}}, "Bearer other_token"), + ({}, None), + ], +) +def test_get_authenticator(config, expected_token): + source = SourceNotion() + authenticator = source._get_authenticator(config) # Fixed line + + if expected_token: + assert isinstance(authenticator, TokenAuthenticator) + assert authenticator.token == expected_token # Replace with the actual way to access the token from the authenticator + else: + assert authenticator is None + + +@pytest.mark.parametrize( + "config, expected_return", + [ + ({}, None), + ({"start_date": "2021-01-01T00:00:00.000Z"}, None), + ({"start_date": "2021-99-99T79:89:99.123Z"}, "The provided start date is not a valid date. Please check the format and try again."), + ({"start_date": "2021-01-01T00:00:00.000"}, "Please check the format of the start date against the pattern descriptor."), + ({"start_date": "2025-01-25T00:00:00.000Z"}, "The start date cannot be greater than the current date."), + ], +) +def test_validate_start_date(config, expected_return): + source = SourceNotion() + result = source._validate_start_date(config) + assert result == expected_return diff --git a/source-notion/unit_tests/test_streams.py b/source-notion/unit_tests/test_streams.py new file mode 100644 index 0000000000..d369b201be --- /dev/null +++ b/source-notion/unit_tests/test_streams.py @@ -0,0 +1,352 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +import random +from http import HTTPStatus +from unittest.mock import MagicMock + +import freezegun +import pytest +import requests +from airbyte_cdk.models import SyncMode +from source_notion.streams import Blocks, NotionStream, Pages, Users + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(NotionStream, "path", "v0/example_endpoint") + mocker.patch.object(NotionStream, "primary_key", "test_primary_key") + mocker.patch.object(NotionStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = NotionStream(config=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class, requests_mock): + stream = NotionStream(config=MagicMock()) + requests_mock.get("https://dummy", json={"next_cursor": "aaa"}) + inputs = {"response": requests.get("https://dummy")} + expected_token = {"next_cursor": "aaa"} + assert stream.next_page_token(**inputs) == expected_token + + +@pytest.mark.parametrize( + "response_json, expected_output", + [({"next_cursor": "some_cursor", "has_more": True}, {"next_cursor": "some_cursor"}), ({"has_more": False}, None), ({}, None)], +) +def test_next_page_token_with_no_cursor(patch_base_class, response_json, expected_output): + stream = NotionStream(config=MagicMock()) + mock_response = MagicMock() + mock_response.json.return_value = response_json + result = stream.next_page_token(mock_response) + assert result == expected_output + + +def test_parse_response(patch_base_class, requests_mock): + stream = NotionStream(config=MagicMock()) + requests_mock.get("https://dummy", json={"results": [{"a": 123}, {"b": "xx"}]}) + resp = requests.get("https://dummy") + inputs = {"response": resp, "stream_state": MagicMock()} + expected_parsed_object = [{"a": 123}, {"b": "xx"}] + assert list(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = NotionStream(config=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {"Notion-Version": "2022-06-28"} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = NotionStream(config=MagicMock()) + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, True), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + (HTTPStatus.BAD_GATEWAY, True), + (HTTPStatus.FORBIDDEN, False), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = NotionStream(config=MagicMock()) + assert stream.should_retry(response_mock) == should_retry + + +def test_should_not_retry_with_ai_block(requests_mock): + stream = Blocks(parent=None, config=MagicMock()) + json_response = { + "object": "error", + "status": 400, + "code": "validation_error", + "message": "Block type ai_block is not supported via the API.", + } + requests_mock.get("https://api.notion.com/v1/blocks/123", json=json_response, status_code=400) + test_response = requests.get("https://api.notion.com/v1/blocks/123") + assert not stream.should_retry(test_response) + + +def test_should_not_retry_with_not_found_block(requests_mock): + stream = Blocks(parent=None, config=MagicMock()) + json_response = { + "object": "error", + "status": 404, + "message": "Not Found for url: https://api.notion.com/v1/blocks/123/children?page_size=100", + } + requests_mock.get("https://api.notion.com/v1/blocks/123", json=json_response, status_code=404) + test_response = requests.get("https://api.notion.com/v1/blocks/123") + assert not stream.should_retry(test_response) + + +def test_empty_blocks_results(requests_mock): + stream = Blocks(parent=None, config=MagicMock()) + requests_mock.get( + "https://api.notion.com/v1/blocks/aaa/children", + json={ + "next_cursor": None, + }, + ) + stream.block_id_stack = ["aaa"] + assert list(stream.read_records(sync_mode=SyncMode.incremental, stream_slice=[])) == [] + + +@pytest.mark.parametrize( + "status_code,retry_after_header,expected_backoff", + [ + (429, "10", 10.0), # Case for 429 error with retry-after header + (429, None, 5.0), # Case for 429 error without retry-after header, should default to 5.0 + (504, None, None), # Case for 500-level error, should default to None and use CDK exponential backoff + (400, None, 10.0), # Case for specific 400-level error handled by check_invalid_start_cursor + ], +) +def test_backoff_time(status_code, retry_after_header, expected_backoff, patch_base_class): + response_mock = MagicMock(spec=requests.Response) + response_mock.status_code = status_code + response_mock.headers = {"retry-after": retry_after_header} if retry_after_header else {} + stream = NotionStream(config=MagicMock()) + + assert stream.backoff_time(response_mock) == expected_backoff + + +def test_users_request_params(patch_base_class): + stream = Users(config=MagicMock()) + + # No next_page_token. First pull + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"page_size": 100} + assert stream.request_params(**inputs) == expected_params + + # When getting pages after the first pull. + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"next_cursor": "123"}} + expected_params = {"start_cursor": "123", "page_size": 100} + assert stream.request_params(**inputs) == expected_params + + +def test_user_stream_handles_pagination_correctly(requests_mock): + """ + Test shows that Users stream uses pagination as per Notion API docs. + """ + + response_body = { + "object": "list", + "results": [{"id": f"{x}", "object": "user", "type": ["person", "bot"][random.randint(0, 1)]} for x in range(100)], + "next_cursor": "bc48234b-77b2-41a6-95a3-6a8abb7887d5", + "has_more": True, + "type": "user", + } + requests_mock.get("https://api.notion.com/v1/users?page_size=100", json=response_body) + + response_body = { + "object": "list", + "results": [{"id": f"{x}", "object": "user", "type": ["person", "bot"][random.randint(0, 1)]} for x in range(100, 200)], + "next_cursor": "67030467-b97b-4729-8fd6-2fb33d012da4", + "has_more": True, + "type": "user", + } + requests_mock.get("https://api.notion.com/v1/users?page_size=100&start_cursor=bc48234b-77b2-41a6-95a3-6a8abb7887d5", json=response_body) + + response_body = { + "object": "list", + "results": [{"id": f"{x}", "object": "user", "type": ["person", "bot"][random.randint(0, 1)]} for x in range(200, 220)], + "next_cursor": None, + "has_more": False, + "type": "user", + } + requests_mock.get("https://api.notion.com/v1/users?page_size=100&start_cursor=67030467-b97b-4729-8fd6-2fb33d012da4", json=response_body) + + stream = Users(config=MagicMock()) + + records = stream.read_records(sync_mode=SyncMode.full_refresh) + records_length = sum(1 for _ in records) + assert records_length == 220 + + +@pytest.mark.parametrize( + "config, expected_start_date, current_time", + [ + ( + {"authenticator": "secret_token", "start_date": "2021-09-01T00:00:00.000Z"}, + "2021-09-01T00:00:00.000Z", + "2022-09-22T00:00:00.000Z", + ), + ({"authenticator": "super_secret_token", "start_date": None}, "2020-09-22T00:00:00.000Z", "2022-09-22T00:00:00.000Z"), + ({"authenticator": "even_more_secret_token"}, "2021-01-01T12:30:00.000Z", "2023-01-01T12:30:00.000Z"), + ], +) +def test_set_start_date(patch_base_class, config, expected_start_date, current_time): + """ + Test that start_date in config is either: + 1. set to the value provided by the user + 2. defaults to two years from the present date set by the test environment. + """ + with freezegun.freeze_time(current_time): + stream = NotionStream(config=config) + assert stream.start_date == expected_start_date + + +@pytest.mark.parametrize( + "stream,parent,url,status_code,response_content,expected_availability,expected_reason_substring", + [ + ( + Users, + None, + "https://api.notion.com/v1/users", + 403, + b'{"object": "error", "status": 403, "code": "restricted_resource"}', + False, + "This is likely due to insufficient permissions for your Notion integration.", + ), + ( + Blocks, + Pages, + "https://api.notion.com/v1/blocks/123/children", + 403, + b'{"object": "error", "status": 403, "code": "restricted_resource"}', + False, + "This is likely due to insufficient permissions for your Notion integration.", + ), + ( + Users, + None, + "https://api.notion.com/v1/users", + 200, + b'{"object": "list", "results": [{"id": "123", "object": "user", "type": "person"}]}', + True, + None, + ), + ], +) +def test_403_error_handling( + requests_mock, stream, parent, url, status_code, response_content, expected_availability, expected_reason_substring +): + """ + Test that availability strategy flags streams with 403 error as unavailable + and returns custom Notion integration message. + """ + + requests_mock.get(url=url, status_code=status_code, content=response_content) + + if parent: + stream = stream(parent=parent, config=MagicMock()) + stream.parent.stream_slices = MagicMock(return_value=[{"id": "123"}]) + stream.parent.read_records = MagicMock(return_value=[{"id": "123", "object": "page"}]) + else: + stream = stream(config=MagicMock()) + + is_available, reason = stream.check_availability(logger=logging.Logger, source=MagicMock()) + + assert is_available is expected_availability + + if expected_reason_substring: + assert expected_reason_substring in reason + else: + assert reason is None + + +@pytest.mark.parametrize( + "initial_page_size, expected_page_size, mock_response", + [ + (100, 50, {"status_code": 504, "json": {}, "headers": {"retry-after": "1"}}), + (50, 25, {"status_code": 504, "json": {}, "headers": {"retry-after": "1"}}), + (100, 100, {"status_code": 429, "json": {}, "headers": {"retry-after": "1"}}), + (50, 100, {"status_code": 200, "json": {"data": "success"}, "headers": {}}), + ], + ids=[ + "504 error, page_size 100 -> 50", + "504 error, page_size 50 -> 25", + "429 error, page_size 100 -> 100", + "200 success, page_size 50 -> 100", + ], +) +def test_request_throttle(initial_page_size, expected_page_size, mock_response, requests_mock): + """ + Tests that the request page_size is halved when a 504 error is encountered. + Once a 200 success is encountered, the page_size is reset to 100, for use in the next call. + """ + requests_mock.register_uri( + "GET", + "https://api.notion.com/v1/users", + [{"status_code": mock_response["status_code"], "json": mock_response["json"], "headers": mock_response["headers"]}], + ) + + stream = Users(config={"authenticator": "auth"}) + stream.page_size = initial_page_size + response = requests.get("https://api.notion.com/v1/users") + + stream.should_retry(response=response) + + assert stream.page_size == expected_page_size + + +def test_users_record_transformer(): + stream = Users(config=MagicMock()) + response_record = { + "object": "user", "id": "id", "name": "Airbyte", "avatar_url": "some url", "type": "bot", + "bot": {"owner": {"type": "user", "user": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", + "person": {"email": "email"}}}, "workspace_name": "test"} + } + expected_record = { + "object": "user", "id": "id", "name": "Airbyte", "avatar_url": "some url", "type": "bot", + "bot": {"owner": {"type": "user", "info": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", + "person": {"email": "email"}}}, "workspace_name": "test"} + } + assert stream.transform(response_record) == expected_record + + +def test_block_record_transformer(): + stream = Blocks(parent=None, config=MagicMock()) + response_record = { + "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", + "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, + {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "mention", "mention": {"type": "page", "page": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, + "plain_text": "test", "href": "https://www.notion.so/id"}], "color": "default"} + } + expected_record = { + "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", + "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations":{"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text":"test", "href": None}, + {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, + {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "mention", "mention": {"type": "page", "info": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": "https://www.notion.so/id"}], + "color": "default"} + } + assert stream.transform(response_record) == expected_record From 6a201c790e739a7882cdd7182b17aa71dfa561ef Mon Sep 17 00:00:00 2001 From: Will Baker Date: Thu, 21 Mar 2024 10:41:26 -0400 Subject: [PATCH 50/96] source-notion: snapshot tests of existing ATF connector --- source-notion/acmeCo/blocks.schema.yaml | 1977 +++++++ source-notion/acmeCo/databases.schema.yaml | 658 +++ source-notion/acmeCo/flow.yaml | 18 + source-notion/acmeCo/pages.schema.yaml | 667 +++ source-notion/acmeCo/users.schema.yaml | 42 + source-notion/config.yaml | 18 + source-notion/test.flow.yaml | 34 + source-notion/tests/__init__.py | 0 .../snapshots__capture__capture.stdout.json | 740 +++ .../snapshots__discover__capture.stdout.json | 5099 +++++++++++++++++ .../snapshots__spec__capture.stdout.json | 148 + source-notion/tests/test_snapshots.py | 62 + 12 files changed, 9463 insertions(+) create mode 100644 source-notion/acmeCo/blocks.schema.yaml create mode 100644 source-notion/acmeCo/databases.schema.yaml create mode 100644 source-notion/acmeCo/flow.yaml create mode 100644 source-notion/acmeCo/pages.schema.yaml create mode 100644 source-notion/acmeCo/users.schema.yaml create mode 100644 source-notion/config.yaml create mode 100644 source-notion/test.flow.yaml create mode 100644 source-notion/tests/__init__.py create mode 100644 source-notion/tests/snapshots/snapshots__capture__capture.stdout.json create mode 100644 source-notion/tests/snapshots/snapshots__discover__capture.stdout.json create mode 100644 source-notion/tests/snapshots/snapshots__spec__capture.stdout.json create mode 100644 source-notion/tests/test_snapshots.py diff --git a/source-notion/acmeCo/blocks.schema.yaml b/source-notion/acmeCo/blocks.schema.yaml new file mode 100644 index 0000000000..11509b8a0f --- /dev/null +++ b/source-notion/acmeCo/blocks.schema.yaml @@ -0,0 +1,1977 @@ +--- +$schema: "http://json-schema.org/draft-07/schema#" +additionalProperties: true +properties: + archived: + type: boolean + bookmark: + properties: + caption: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + url: + type: string + type: object + bulleted_list_item: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + children: + items: + type: object + type: array + color: + type: string + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: object + callout: + properties: + children: + items: + type: object + type: array + color: + type: string + icon: + $schema: "http://json-schema.org/draft-07/schema#" + anyOf: + - $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + caption: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: + - "null" + - array + external: + additionalProperties: true + properties: + url: + type: string + type: + - "null" + - object + file: + additionalProperties: true + properties: + expiry_time: + type: string + url: + type: string + type: + - "null" + - object + type: + enum: + - file + - external + type: + - "null" + - object + - $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + emoji: + type: string + type: + type: string + type: + - "null" + - object + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: object + child_database: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + title: + type: string + type: object + child_page: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + title: + type: string + type: object + code: + properties: + color: + type: string + language: + type: string + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: object + created_time: + type: string + embed: + properties: + url: + type: string + type: object + equation: + properties: + expression: + type: string + type: object + file: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + caption: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: + - "null" + - array + external: + additionalProperties: true + properties: + url: + type: string + type: + - "null" + - object + file: + additionalProperties: true + properties: + expiry_time: + type: string + url: + type: string + type: + - "null" + - object + type: + type: string + type: + - "null" + - object + has_children: + type: + - "null" + - boolean + heading_1: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + color: + type: string + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: + - "null" + - object + heading_2: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + color: + type: string + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: + - "null" + - object + heading_3: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + color: + type: string + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: + - "null" + - object + id: + type: string + image: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + caption: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: + - "null" + - array + external: + additionalProperties: true + properties: + url: + type: string + type: + - "null" + - object + file: + additionalProperties: true + properties: + expiry_time: + type: string + url: + type: string + type: + - "null" + - object + type: + type: string + type: + - "null" + - object + last_edited_time: + type: string + numbered_list_item: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + children: + items: + type: object + type: array + color: + type: string + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: object + object: + type: string + paragraph: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + children: + items: + type: object + type: array + color: + type: string + rich_text: + items: + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: object + pdf: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + caption: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: + - "null" + - array + external: + additionalProperties: true + properties: + url: + type: string + type: + - "null" + - object + file: + additionalProperties: true + properties: + expiry_time: + type: string + url: + type: string + type: + - "null" + - object + type: + type: string + type: + - "null" + - object + quote: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + children: + items: + type: object + type: array + color: + type: string + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: object + to_do: + properties: + checked: + type: + - "null" + - boolean + children: + items: + type: object + type: array + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: object + toggle: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + children: + items: + type: object + type: array + color: + type: string + text: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + type: object + type: + type: string + vidoe: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + caption: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: + - "null" + - array + external: + additionalProperties: true + properties: + url: + type: string + type: + - "null" + - object + file: + additionalProperties: true + properties: + expiry_time: + type: string + url: + type: string + type: + - "null" + - object + type: + type: string + type: + - "null" + - object +required: + - id +type: object diff --git a/source-notion/acmeCo/databases.schema.yaml b/source-notion/acmeCo/databases.schema.yaml new file mode 100644 index 0000000000..0a1d69ef37 --- /dev/null +++ b/source-notion/acmeCo/databases.schema.yaml @@ -0,0 +1,658 @@ +--- +$defs: + options.json: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + color: + enum: + - default + - gray + - brown + - orange + - yellow + - green + - blue + - purple + - pink + - red + id: + type: string + name: + type: string + type: + - "null" + - object +$schema: "http://json-schema.org/draft-07/schema#" +additionalProperties: true +properties: + archived: + type: boolean + cover: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + caption: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: + - "null" + - array + external: + additionalProperties: true + properties: + url: + type: string + type: + - "null" + - object + file: + additionalProperties: true + properties: + expiry_time: + type: string + url: + type: string + type: + - "null" + - object + type: + type: string + type: + - "null" + - object + created_by: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + avatar_url: + type: + - "null" + - string + bot: + additionalProperties: true + properties: + owner: + properties: + type: + type: string + workspace: + type: + - "null" + - boolean + type: object + type: + - "null" + - object + id: + type: string + name: + type: string + object: + type: string + person: + additionalProperties: true + properties: + email: + type: string + type: + - "null" + - object + type: + type: string + type: object + created_time: + type: string + icon: + $schema: "http://json-schema.org/draft-07/schema#" + anyOf: + - $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + caption: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: + - "null" + - array + external: + additionalProperties: true + properties: + url: + type: string + type: + - "null" + - object + file: + additionalProperties: true + properties: + expiry_time: + type: string + url: + type: string + type: + - "null" + - object + type: + enum: + - file + - external + type: + - "null" + - object + - $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + emoji: + type: string + type: + type: string + type: + - "null" + - object + id: + type: string + is_inline: + type: + - "null" + - boolean + last_edited_by: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + avatar_url: + type: + - "null" + - string + bot: + additionalProperties: true + properties: + owner: + properties: + type: + type: string + workspace: + type: + - "null" + - boolean + type: object + type: + - "null" + - object + id: + type: string + name: + type: string + object: + type: string + person: + additionalProperties: true + properties: + email: + type: string + type: + - "null" + - object + type: + type: string + type: object + last_edited_time: + type: string + object: + type: string + parent: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + database_id: + type: string + page_id: + type: string + type: + type: string + workspace: + type: boolean + type: object + properties: + items: + additionalProperties: true + properties: + name: + type: + - "null" + - string + value: + additionalProperties: true + anyOf: + - properties: + id: + type: string + type: + type: string + type: object + - additionalProperties: true + properties: + id: + type: string + name: + type: string + type: + enum: + - title + - rich_text + - date + - people + - files + - checkbox + - url + - email + - phone_number + - created_time + - created_by + - last_edited_time + - last_edited_by + type: object + - properties: + format: + enum: + - number + - number_with_commas + - percent + - dollar + - canadian_dollar + - euro + - pound + - yen + - ruble + - rupee + - won + - yuan + - real + - lira + - rupiah + - franc + - hong_kong_dollar + - new_zealand_dollar + - krona + - norwegian_krone + - mexican_peso + - rand + - new_taiwan_dollar + - danish_krone + - zloty + - baht + - forint + - koruna + - shekel + - chilean_peso + - philippine_peso + - dirham + - colombian_peso + - riyal + - ringgit + - leu + id: + type: string + name: + type: string + type: + enum: + - number + type: object + - properties: + id: + type: string + name: + type: string + options: + items: + $ref: "#/$defs/options.json" + type: array + type: + enum: + - select + - multi_select + type: object + - properties: + expression: + type: string + id: + type: string + name: + type: string + type: + enum: + - formula + type: object + - properties: + database_id: + type: string + id: + type: string + name: + type: string + synced_property_id: + type: + - "null" + - string + synced_property_name: + type: + - "null" + - string + type: + enum: + - relation + type: object + - properties: + function: + enum: + - count_all + - count_values + - count_unique_values + - count_empty + - count_not_empty + - percent_empty + - percent_not_empty + - sum + - average + - median + - min + - max + - range + - show_original + id: + type: string + name: + type: string + relation_property_id: + type: string + relation_property_name: + type: string + rollup_property_id: + type: string + rollup_property_name: + type: string + type: + enum: + - rollup + type: object + type: object + type: + - "null" + - object + type: array + title: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: array + url: + type: string +required: + - id +type: object diff --git a/source-notion/acmeCo/flow.yaml b/source-notion/acmeCo/flow.yaml new file mode 100644 index 0000000000..29c9f9a470 --- /dev/null +++ b/source-notion/acmeCo/flow.yaml @@ -0,0 +1,18 @@ +--- +collections: + acmeCo/blocks: + schema: blocks.schema.yaml + key: + - /id + acmeCo/databases: + schema: databases.schema.yaml + key: + - /id + acmeCo/pages: + schema: pages.schema.yaml + key: + - /id + acmeCo/users: + schema: users.schema.yaml + key: + - /id diff --git a/source-notion/acmeCo/pages.schema.yaml b/source-notion/acmeCo/pages.schema.yaml new file mode 100644 index 0000000000..934550d015 --- /dev/null +++ b/source-notion/acmeCo/pages.schema.yaml @@ -0,0 +1,667 @@ +--- +$defs: + date.json: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + end: + type: + - "null" + - string + start: + type: + - "null" + - string + time_zone: + type: + - "null" + - string + type: + - "null" + - object + options.json: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + color: + enum: + - default + - gray + - brown + - orange + - yellow + - green + - blue + - purple + - pink + - red + id: + type: string + name: + type: string + type: + - "null" + - object + rich_text.json: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + title.json: + items: + additionalProperties: true + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + text: + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - array + user.json: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + avatar_url: + type: + - "null" + - string + bot: + additionalProperties: true + properties: + owner: + properties: + type: + type: string + workspace: + type: + - "null" + - boolean + type: object + type: + - "null" + - object + id: + type: string + name: + type: string + object: + enum: + - user + person: + additionalProperties: true + properties: + email: + type: string + type: + - "null" + - object + type: + enum: + - person + - bot + type: object +$schema: "http://json-schema.org/draft-07/schema#" +additionalProperties: true +properties: + archived: + type: boolean + cover: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + caption: + items: + $schema: "http://json-schema.org/draft-07/schema#" + properties: + annotations: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + type: + - "null" + - object + href: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + rich_text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + text: + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + additionalProperties: true + properties: + type: + type: string + url: + type: + - "null" + - string + type: + - "null" + - object + type: + - "null" + - object + type: + type: + - "null" + - string + type: object + type: + - "null" + - array + external: + additionalProperties: true + properties: + url: + type: string + type: + - "null" + - object + file: + additionalProperties: true + properties: + expiry_time: + type: string + url: + type: string + type: + - "null" + - object + type: + type: string + type: + - "null" + - object + created_time: + type: string + id: + type: string + last_edited_time: + type: string + object: + type: string + parent: + $schema: "http://json-schema.org/draft-07/schema#" + additionalProperties: true + properties: + database_id: + type: string + page_id: + type: string + type: + type: string + workspace: + type: boolean + type: object + properties: + items: + additionalProperties: true + properties: + name: + type: + - "null" + - string + value: + additionalProperties: true + anyOf: + - properties: + id: + type: string + type: + type: string + type: object + - properties: + id: + type: string + title: + $ref: "#/$defs/title.json" + type: + enum: + - title + type: object + - properties: + id: + type: string + rich_text: + $ref: "#/$defs/rich_text.json" + type: + enum: + - rich_text + type: object + - properties: + id: + type: string + select: + $ref: "#/$defs/options.json" + type: + enum: + - select + type: object + - properties: + id: + type: string + multi_select: + items: + $ref: "#/$defs/options.json" + type: + - "null" + - array + type: + enum: + - multi_select + type: object + - properties: + date: + $ref: "#/$defs/date.json" + id: + type: string + type: + enum: + - date + type: object + - properties: + formula: + properties: + boolean: + type: + - "null" + - boolean + date: + $ref: "#/$defs/date.json" + number: + type: + - "null" + - number + string: + type: + - "null" + - string + type: + enum: + - string + - number + - boolean + - date + type: + - "null" + - object + id: + type: string + type: + enum: + - formula + type: object + - properties: + id: + type: string + relation: + items: + properties: + id: + type: string + type: object + type: + - "null" + - array + type: + enum: + - relation + type: object + - properties: + id: + type: string + rollup: + properties: + array: + items: + properties: + type: + type: string + type: object + type: + - "null" + - array + date: + $ref: "#/$defs/date.json" + number: + type: + - "null" + - number + type: + enum: + - number + - date + - array + type: + - "null" + - object + type: + enum: + - rollup + type: object + - properties: + id: + type: string + people: + items: + $ref: "#/$defs/user.json" + type: + - "null" + - array + type: + enum: + - people + type: object + - properties: + files: + items: + properties: + expiry_time: + type: + - "null" + - string + name: + type: string + type: + enum: + - external + - file + url: + type: string + type: object + type: + - "null" + - array + id: + type: string + type: + enum: + - files + type: object + - properties: + checkout: + type: + - "null" + - boolean + id: + type: string + type: + enum: + - checkbox + type: object + - properties: + id: + type: string + type: + enum: + - url + url: + type: string + type: object + - properties: + email: + type: string + id: + type: string + type: + enum: + - email + type: object + - properties: + id: + type: string + phone_number: + type: string + type: + enum: + - phone_number + type: object + - properties: + created_time: + type: string + id: + type: string + type: + enum: + - created_time + type: object + - properties: + created_by: + $ref: "#/$defs/user.json" + id: + type: string + type: + enum: + - created_by + type: object + - properties: + id: + type: string + last_edited_time: + type: string + type: + enum: + - last_edited_time + type: object + - properties: + id: + type: string + last_edited_by: + $ref: "#/$defs/user.json" + type: + enum: + - last_edited_by + type: object + type: object + type: object + type: array + url: + type: string +required: + - id +type: object diff --git a/source-notion/acmeCo/users.schema.yaml b/source-notion/acmeCo/users.schema.yaml new file mode 100644 index 0000000000..9e85d079e1 --- /dev/null +++ b/source-notion/acmeCo/users.schema.yaml @@ -0,0 +1,42 @@ +--- +$schema: "http://json-schema.org/draft-07/schema#" +additionalProperties: true +properties: + avatar_url: + type: + - "null" + - string + bot: + additionalProperties: true + properties: + owner: + properties: + type: + type: string + workspace: + type: + - "null" + - boolean + type: object + type: + - "null" + - object + id: + type: string + name: + type: string + object: + type: string + person: + additionalProperties: true + properties: + email: + type: string + type: + - "null" + - object + type: + type: string +required: + - id +type: object diff --git a/source-notion/config.yaml b/source-notion/config.yaml new file mode 100644 index 0000000000..279a972a44 --- /dev/null +++ b/source-notion/config.yaml @@ -0,0 +1,18 @@ +credentials: + auth_type: token + token_sops: ENC[AES256_GCM,data:JLvGUfdIfvDf0qI2/Zr/EDSIP0J9gTxkZ1E7h++YdUQ2l+HYkt+jyCPxwA0ZngYgSKA=,iv:o7v32H+CNTG1m2ye/Je8EqiO3LIqayu3XjoqVMhvJJQ=,tag:if2G5KH3UdJSw6FZXae9kQ==,type:str] +start_date: "2023-04-17T00:00:00.000Z" +sops: + kms: [] + gcp_kms: + - resource_id: projects/estuary-theatre/locations/us-central1/keyRings/connector-keyring/cryptoKeys/connector-repository + created_at: "2024-03-21T14:36:06Z" + enc: CiQAdmEdwo9a6M+bUq8DMIWi5qxu279bHJSyQEzHzwjDAG8PTpMSSQCSobQjpCxFWDU3orGpaPl5eKe3slAxPYNZZofFGkHTytinGYQOLwziGp4/ImDjjoCzM0/S42sipLERThX0g6X9JTApvtEQCzI= + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-03-21T14:36:07Z" + mac: ENC[AES256_GCM,data:mUkcMuguGP2nr1Ybc3lGpltlzQs8nZUoD4rjgB4YF1lx8z6k1sH859/vkgWYhjK1z6iXxQVuGOVQkzwOUHwRH1q+gNVvdKmXM2evrVoDzeXyjFvQNAQsWHRlRSNh+Tz0nMQHOAw8Z3WRNVsUQ4QjztzPLjaPt5Z0+J1qrayU4GE=,iv:9vv9w5j66Tc651C5iDAeWfOYjia6T65Wjxmp/m+rWGk=,tag:u1oTx00Zh6ov03smk/zdVQ==,type:str] + pgp: [] + encrypted_suffix: _sops + version: 3.8.1 diff --git a/source-notion/test.flow.yaml b/source-notion/test.flow.yaml new file mode 100644 index 0000000000..72f1c1964c --- /dev/null +++ b/source-notion/test.flow.yaml @@ -0,0 +1,34 @@ +--- +import: + - acmeCo/flow.yaml +captures: + acmeCo/source-notion: + endpoint: + connector: + image: "ghcr.io/estuary/source-notion:v2" + config: config.yaml + bindings: + - resource: + stream: users + syncMode: full_refresh + cursorField: + - id + target: acmeCo/users + - resource: + stream: databases + syncMode: incremental + cursorField: + - last_edited_time + target: acmeCo/databases + - resource: + stream: pages + syncMode: incremental + cursorField: + - last_edited_time + target: acmeCo/pages + - resource: + stream: blocks + syncMode: incremental + cursorField: + - last_edited_time + target: acmeCo/blocks diff --git a/source-notion/tests/__init__.py b/source-notion/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source-notion/tests/snapshots/snapshots__capture__capture.stdout.json b/source-notion/tests/snapshots/snapshots__capture__capture.stdout.json new file mode 100644 index 0000000000..b0ed06fb0a --- /dev/null +++ b/source-notion/tests/snapshots/snapshots__capture__capture.stdout.json @@ -0,0 +1,740 @@ +[ + [ + "acmeCo/users", + { + "avatar_url": null, + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "name": "WillB Estuary", + "object": "user", + "person": {}, + "type": "person" + } + ], + [ + "acmeCo/users", + { + "avatar_url": null, + "bot": { + "owner": { + "type": "workspace", + "workspace": true + }, + "workspace_name": "WillB Estuary's Notion" + }, + "id": "e1440030-aafa-458b-b03e-854a8e2fc629", + "name": "estuary", + "object": "user", + "type": "bot" + } + ], + [ + "acmeCo/databases", + { + "archived": false, + "cover": null, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:42:00.000Z", + "description": [], + "icon": null, + "id": "20e1921f-a184-4b40-bfb1-513ad5bb90af", + "is_inline": true, + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:43:00.000Z", + "object": "database", + "parent": { + "page_id": "45c1fbd9-2cc9-4160-93a4-3ebb94f77914", + "type": "page_id" + }, + "properties": [ + { + "name": "value", + "value": { + "id": "Mw%3F_", + "name": "value", + "number": { + "format": "number" + }, + "type": "number" + } + }, + { + "name": "Tags", + "value": { + "id": "g%5C%3Fo", + "multi_select": { + "options": [ + { + "color": "pink", + "description": null, + "id": "15d458c9-e810-4487-a1b5-35aabd8be0a3", + "name": "myTag" + } + ] + }, + "name": "Tags", + "type": "multi_select" + } + }, + { + "name": "Name", + "value": { + "id": "title", + "name": "Name", + "title": {}, + "type": "title" + } + } + ], + "public_url": null, + "title": [ + { + "annotations": { + "bold": false, + "code": false, + "color": "default", + "italic": false, + "strikethrough": false, + "underline": false + }, + "href": null, + "plain_text": "database", + "text": { + "content": "database", + "link": null + }, + "type": "text" + } + ], + "url": "https://www.notion.so/20e1921fa1844b40bfb1513ad5bb90af" + } + ], + [ + "acmeCo/pages", + { + "archived": false, + "cover": null, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:42:00.000Z", + "icon": null, + "id": "044986df-d173-437c-9bf9-014296e5d4d2", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:42:00.000Z", + "object": "page", + "parent": { + "database_id": "20e1921f-a184-4b40-bfb1-513ad5bb90af", + "type": "database_id" + }, + "properties": [ + { + "name": "value", + "value": { + "id": "Mw%3F_", + "number": null, + "type": "number" + } + }, + { + "name": "Tags", + "value": { + "id": "g%5C%3Fo", + "multi_select": [], + "type": "multi_select" + } + }, + { + "name": "Name", + "value": { + "id": "title", + "title": [], + "type": "title" + } + } + ], + "public_url": null, + "url": "https://www.notion.so/044986dfd173437c9bf9014296e5d4d2" + } + ], + [ + "acmeCo/pages", + { + "archived": false, + "cover": null, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:27:00.000Z", + "icon": null, + "id": "45c1fbd9-2cc9-4160-93a4-3ebb94f77914", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:42:00.000Z", + "object": "page", + "parent": { + "type": "workspace", + "workspace": true + }, + "properties": [ + { + "name": "title", + "value": { + "id": "title", + "title": [ + { + "annotations": { + "bold": false, + "code": false, + "color": "default", + "italic": false, + "strikethrough": false, + "underline": false + }, + "href": null, + "plain_text": "some blocks", + "text": { + "content": "some blocks", + "link": null + }, + "type": "text" + } + ], + "type": "title" + } + } + ], + "public_url": null, + "url": "https://www.notion.so/some-blocks-45c1fbd92cc9416093a43ebb94f77914" + } + ], + [ + "acmeCo/pages", + { + "archived": false, + "cover": null, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:42:00.000Z", + "icon": null, + "id": "9291c59d-f8dd-4af8-a37a-a6646988c5d2", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T14:45:00.000Z", + "object": "page", + "parent": { + "database_id": "20e1921f-a184-4b40-bfb1-513ad5bb90af", + "type": "database_id" + }, + "properties": [ + { + "name": "value", + "value": { + "id": "Mw%3F_", + "number": 24, + "type": "number" + } + }, + { + "name": "Tags", + "value": { + "id": "g%5C%3Fo", + "multi_select": [], + "type": "multi_select" + } + }, + { + "name": "Name", + "value": { + "id": "title", + "title": [ + { + "annotations": { + "bold": false, + "code": false, + "color": "default", + "italic": false, + "strikethrough": false, + "underline": false + }, + "href": null, + "plain_text": "second row", + "text": { + "content": "second row", + "link": null + }, + "type": "text" + } + ], + "type": "title" + } + } + ], + "public_url": null, + "url": "https://www.notion.so/second-row-9291c59df8dd4af8a37aa6646988c5d2" + } + ], + [ + "acmeCo/pages", + { + "archived": false, + "cover": null, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:42:00.000Z", + "icon": null, + "id": "a5417102-7101-4972-aa35-eda60453e20c", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:42:00.000Z", + "object": "page", + "parent": { + "page_id": "45c1fbd9-2cc9-4160-93a4-3ebb94f77914", + "type": "page_id" + }, + "properties": [ + { + "name": "title", + "value": { + "id": "title", + "title": [ + { + "annotations": { + "bold": false, + "code": false, + "color": "default", + "italic": false, + "strikethrough": false, + "underline": false + }, + "href": null, + "plain_text": "embedded page", + "text": { + "content": "embedded page", + "link": null + }, + "type": "text" + } + ], + "type": "title" + } + } + ], + "public_url": null, + "url": "https://www.notion.so/embedded-page-a541710271014972aa35eda60453e20c" + } + ], + [ + "acmeCo/pages", + { + "archived": false, + "cover": null, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:42:00.000Z", + "icon": null, + "id": "cc5f4018-24ed-4300-9e1d-440d9bd43676", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:43:00.000Z", + "object": "page", + "parent": { + "database_id": "20e1921f-a184-4b40-bfb1-513ad5bb90af", + "type": "database_id" + }, + "properties": [ + { + "name": "value", + "value": { + "id": "Mw%3F_", + "number": 12, + "type": "number" + } + }, + { + "name": "Tags", + "value": { + "id": "g%5C%3Fo", + "multi_select": [ + { + "color": "pink", + "id": "15d458c9-e810-4487-a1b5-35aabd8be0a3", + "name": "myTag" + } + ], + "type": "multi_select" + } + }, + { + "name": "Name", + "value": { + "id": "title", + "title": [ + { + "annotations": { + "bold": false, + "code": false, + "color": "default", + "italic": false, + "strikethrough": false, + "underline": false + }, + "href": null, + "plain_text": "first row", + "text": { + "content": "first row", + "link": null + }, + "type": "text" + } + ], + "type": "title" + } + } + ], + "public_url": null, + "url": "https://www.notion.so/first-row-cc5f401824ed43009e1d440d9bd43676" + } + ], + [ + "acmeCo/blocks", + { + "archived": false, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:42:00.000Z", + "has_children": false, + "id": "1aa059d4-c48d-4655-af76-4a59b6151935", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:42:00.000Z", + "object": "block", + "paragraph": { + "color": "default", + "rich_text": [] + }, + "parent": { + "page_id": "a5417102-7101-4972-aa35-eda60453e20c", + "type": "page_id" + }, + "type": "paragraph" + } + ], + [ + "acmeCo/blocks", + { + "archived": false, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:27:00.000Z", + "has_children": false, + "id": "27ac18f9-d494-4801-b675-05c283120955", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:28:00.000Z", + "object": "block", + "paragraph": { + "color": "default", + "rich_text": [ + { + "annotations": { + "bold": false, + "code": false, + "color": "default", + "italic": false, + "strikethrough": false, + "underline": false + }, + "href": null, + "plain_text": "text block", + "text": { + "content": "text block", + "link": null + }, + "type": "text" + } + ] + }, + "parent": { + "page_id": "45c1fbd9-2cc9-4160-93a4-3ebb94f77914", + "type": "page_id" + }, + "type": "paragraph" + } + ], + [ + "acmeCo/blocks", + { + "archived": false, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:28:00.000Z", + "has_children": false, + "id": "7dfaf5e3-bbce-4ba9-a531-a1ba3058aa9c", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:28:00.000Z", + "object": "block", + "parent": { + "page_id": "45c1fbd9-2cc9-4160-93a4-3ebb94f77914", + "type": "page_id" + }, + "to_do": { + "checked": false, + "color": "default", + "rich_text": [ + { + "annotations": { + "bold": false, + "code": false, + "color": "default", + "italic": false, + "strikethrough": false, + "underline": false + }, + "href": null, + "plain_text": "Todo 2", + "text": { + "content": "Todo 2", + "link": null + }, + "type": "text" + } + ] + }, + "type": "to_do" + } + ], + [ + "acmeCo/blocks", + { + "archived": false, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:42:00.000Z", + "has_children": false, + "id": "7ef9f951-07e3-4c0e-a242-227c126e597c", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:42:00.000Z", + "object": "block", + "paragraph": { + "color": "default", + "rich_text": [ + { + "annotations": { + "bold": false, + "code": false, + "color": "default", + "italic": false, + "strikethrough": false, + "underline": false + }, + "href": null, + "plain_text": "this is an embedded page", + "text": { + "content": "this is an embedded page", + "link": null + }, + "type": "text" + } + ] + }, + "parent": { + "page_id": "a5417102-7101-4972-aa35-eda60453e20c", + "type": "page_id" + }, + "type": "paragraph" + } + ], + [ + "acmeCo/blocks", + { + "archived": false, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:28:00.000Z", + "has_children": false, + "id": "aea6d5e3-5dc7-412f-b8c7-24a31e9f8a21", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:28:00.000Z", + "object": "block", + "parent": { + "page_id": "45c1fbd9-2cc9-4160-93a4-3ebb94f77914", + "type": "page_id" + }, + "to_do": { + "checked": false, + "color": "default", + "rich_text": [ + { + "annotations": { + "bold": false, + "code": false, + "color": "default", + "italic": false, + "strikethrough": false, + "underline": false + }, + "href": null, + "plain_text": "Todo 1", + "text": { + "content": "Todo 1", + "link": null + }, + "type": "text" + } + ] + }, + "type": "to_do" + } + ], + [ + "acmeCo/blocks", + { + "archived": false, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:42:00.000Z", + "has_children": false, + "id": "bc7b782c-0c7a-480a-b664-f4eb9b58a220", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:42:00.000Z", + "object": "block", + "paragraph": { + "color": "default", + "rich_text": [] + }, + "parent": { + "page_id": "45c1fbd9-2cc9-4160-93a4-3ebb94f77914", + "type": "page_id" + }, + "type": "paragraph" + } + ], + [ + "acmeCo/blocks", + { + "archived": false, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:42:00.000Z", + "has_children": false, + "id": "e15834f9-c267-47f6-b804-a636456559ad", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:42:00.000Z", + "object": "block", + "paragraph": { + "color": "default", + "rich_text": [] + }, + "parent": { + "page_id": "45c1fbd9-2cc9-4160-93a4-3ebb94f77914", + "type": "page_id" + }, + "type": "paragraph" + } + ], + [ + "acmeCo/blocks", + { + "archived": false, + "code": { + "caption": [], + "language": "python", + "rich_text": [ + { + "annotations": { + "bold": false, + "code": false, + "color": "default", + "italic": false, + "strikethrough": false, + "underline": false + }, + "href": null, + "plain_text": "code block", + "text": { + "content": "code block", + "link": null + }, + "type": "text" + } + ] + }, + "created_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "created_time": "2024-03-21T13:28:00.000Z", + "has_children": false, + "id": "e292f784-aa6b-400f-923a-30be2b4a1299", + "last_edited_by": { + "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", + "object": "user" + }, + "last_edited_time": "2024-03-21T13:28:00.000Z", + "object": "block", + "parent": { + "page_id": "45c1fbd9-2cc9-4160-93a4-3ebb94f77914", + "type": "page_id" + }, + "type": "code" + } + ] +] diff --git a/source-notion/tests/snapshots/snapshots__discover__capture.stdout.json b/source-notion/tests/snapshots/snapshots__discover__capture.stdout.json new file mode 100644 index 0000000000..df88879037 --- /dev/null +++ b/source-notion/tests/snapshots/snapshots__discover__capture.stdout.json @@ -0,0 +1,5099 @@ +[ + { + "recommendedName": "users", + "resourceConfig": { + "stream": "users", + "syncMode": "full_refresh", + "cursorField": [ + "id" + ] + }, + "documentSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "bot": { + "additionalProperties": true, + "properties": { + "owner": { + "properties": { + "type": { + "type": "string" + }, + "workspace": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": "object" + } + }, + "type": [ + "null", + "object" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "object": { + "type": "string" + }, + "person": { + "additionalProperties": true, + "properties": { + "email": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": "string" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "key": [ + "/id" + ] + }, + { + "recommendedName": "databases", + "resourceConfig": { + "stream": "databases", + "syncMode": "incremental", + "cursorField": [ + "last_edited_time" + ] + }, + "documentSchema": { + "$defs": { + "options.json": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "color": { + "enum": [ + "default", + "gray", + "brown", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "red" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + } + }, + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "archived": { + "type": "boolean" + }, + "cover": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "caption": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "external": { + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "file": { + "additionalProperties": true, + "properties": { + "expiry_time": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "created_by": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "bot": { + "additionalProperties": true, + "properties": { + "owner": { + "properties": { + "type": { + "type": "string" + }, + "workspace": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": "object" + } + }, + "type": [ + "null", + "object" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "object": { + "type": "string" + }, + "person": { + "additionalProperties": true, + "properties": { + "email": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "created_time": { + "type": "string" + }, + "icon": { + "$schema": "http://json-schema.org/draft-07/schema#", + "anyOf": [ + { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "caption": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "external": { + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "file": { + "additionalProperties": true, + "properties": { + "expiry_time": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "enum": [ + "file", + "external" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "emoji": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + } + ] + }, + "id": { + "type": "string" + }, + "is_inline": { + "type": [ + "null", + "boolean" + ] + }, + "last_edited_by": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "bot": { + "additionalProperties": true, + "properties": { + "owner": { + "properties": { + "type": { + "type": "string" + }, + "workspace": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": "object" + } + }, + "type": [ + "null", + "object" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "object": { + "type": "string" + }, + "person": { + "additionalProperties": true, + "properties": { + "email": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "last_edited_time": { + "type": "string" + }, + "object": { + "type": "string" + }, + "parent": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "database_id": { + "type": "string" + }, + "page_id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "workspace": { + "type": "boolean" + } + }, + "type": "object" + }, + "properties": { + "items": { + "additionalProperties": true, + "properties": { + "name": { + "type": [ + "null", + "string" + ] + }, + "value": { + "additionalProperties": true, + "anyOf": [ + { + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "enum": [ + "title", + "rich_text", + "date", + "people", + "files", + "checkbox", + "url", + "email", + "phone_number", + "created_time", + "created_by", + "last_edited_time", + "last_edited_by" + ] + } + }, + "type": "object" + }, + { + "properties": { + "format": { + "enum": [ + "number", + "number_with_commas", + "percent", + "dollar", + "canadian_dollar", + "euro", + "pound", + "yen", + "ruble", + "rupee", + "won", + "yuan", + "real", + "lira", + "rupiah", + "franc", + "hong_kong_dollar", + "new_zealand_dollar", + "krona", + "norwegian_krone", + "mexican_peso", + "rand", + "new_taiwan_dollar", + "danish_krone", + "zloty", + "baht", + "forint", + "koruna", + "shekel", + "chilean_peso", + "philippine_peso", + "dirham", + "colombian_peso", + "riyal", + "ringgit", + "leu" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "enum": [ + "number" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "options": { + "items": { + "$ref": "#/$defs/options.json" + }, + "type": "array" + }, + "type": { + "enum": [ + "select", + "multi_select" + ] + } + }, + "type": "object" + }, + { + "properties": { + "expression": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "enum": [ + "formula" + ] + } + }, + "type": "object" + }, + { + "properties": { + "database_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "synced_property_id": { + "type": [ + "null", + "string" + ] + }, + "synced_property_name": { + "type": [ + "null", + "string" + ] + }, + "type": { + "enum": [ + "relation" + ] + } + }, + "type": "object" + }, + { + "properties": { + "function": { + "enum": [ + "count_all", + "count_values", + "count_unique_values", + "count_empty", + "count_not_empty", + "percent_empty", + "percent_not_empty", + "sum", + "average", + "median", + "min", + "max", + "range", + "show_original" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "relation_property_id": { + "type": "string" + }, + "relation_property_name": { + "type": "string" + }, + "rollup_property_id": { + "type": "string" + }, + "rollup_property_name": { + "type": "string" + }, + "type": { + "enum": [ + "rollup" + ] + } + }, + "type": "object" + } + ], + "type": "object" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": "array" + }, + "title": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "key": [ + "/id" + ] + }, + { + "recommendedName": "pages", + "resourceConfig": { + "stream": "pages", + "syncMode": "incremental", + "cursorField": [ + "last_edited_time" + ] + }, + "documentSchema": { + "$defs": { + "date.json": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "end": { + "type": [ + "null", + "string" + ] + }, + "start": { + "type": [ + "null", + "string" + ] + }, + "time_zone": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "options.json": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "color": { + "enum": [ + "default", + "gray", + "brown", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "red" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "rich_text.json": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "title.json": { + "items": { + "additionalProperties": true, + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "text": { + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + }, + "user.json": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "bot": { + "additionalProperties": true, + "properties": { + "owner": { + "properties": { + "type": { + "type": "string" + }, + "workspace": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": "object" + } + }, + "type": [ + "null", + "object" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "object": { + "enum": [ + "user" + ] + }, + "person": { + "additionalProperties": true, + "properties": { + "email": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "enum": [ + "person", + "bot" + ] + } + }, + "type": "object" + } + }, + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "archived": { + "type": "boolean" + }, + "cover": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "caption": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "external": { + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "file": { + "additionalProperties": true, + "properties": { + "expiry_time": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "created_time": { + "type": "string" + }, + "id": { + "type": "string" + }, + "last_edited_time": { + "type": "string" + }, + "object": { + "type": "string" + }, + "parent": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "database_id": { + "type": "string" + }, + "page_id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "workspace": { + "type": "boolean" + } + }, + "type": "object" + }, + "properties": { + "items": { + "additionalProperties": true, + "properties": { + "name": { + "type": [ + "null", + "string" + ] + }, + "value": { + "additionalProperties": true, + "anyOf": [ + { + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "title": { + "$ref": "#/$defs/title.json" + }, + "type": { + "enum": [ + "title" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "rich_text": { + "$ref": "#/$defs/rich_text.json" + }, + "type": { + "enum": [ + "rich_text" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "select": { + "$ref": "#/$defs/options.json" + }, + "type": { + "enum": [ + "select" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "multi_select": { + "items": { + "$ref": "#/$defs/options.json" + }, + "type": [ + "null", + "array" + ] + }, + "type": { + "enum": [ + "multi_select" + ] + } + }, + "type": "object" + }, + { + "properties": { + "date": { + "$ref": "#/$defs/date.json" + }, + "id": { + "type": "string" + }, + "type": { + "enum": [ + "date" + ] + } + }, + "type": "object" + }, + { + "properties": { + "formula": { + "properties": { + "boolean": { + "type": [ + "null", + "boolean" + ] + }, + "date": { + "$ref": "#/$defs/date.json" + }, + "number": { + "type": [ + "null", + "number" + ] + }, + "string": { + "type": [ + "null", + "string" + ] + }, + "type": { + "enum": [ + "string", + "number", + "boolean", + "date" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "id": { + "type": "string" + }, + "type": { + "enum": [ + "formula" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "relation": { + "items": { + "properties": { + "id": { + "type": "string" + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "type": { + "enum": [ + "relation" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "rollup": { + "properties": { + "array": { + "items": { + "properties": { + "type": { + "type": "string" + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "date": { + "$ref": "#/$defs/date.json" + }, + "number": { + "type": [ + "null", + "number" + ] + }, + "type": { + "enum": [ + "number", + "date", + "array" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "enum": [ + "rollup" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "people": { + "items": { + "$ref": "#/$defs/user.json" + }, + "type": [ + "null", + "array" + ] + }, + "type": { + "enum": [ + "people" + ] + } + }, + "type": "object" + }, + { + "properties": { + "files": { + "items": { + "properties": { + "expiry_time": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": "string" + }, + "type": { + "enum": [ + "external", + "file" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "id": { + "type": "string" + }, + "type": { + "enum": [ + "files" + ] + } + }, + "type": "object" + }, + { + "properties": { + "checkout": { + "type": [ + "null", + "boolean" + ] + }, + "id": { + "type": "string" + }, + "type": { + "enum": [ + "checkbox" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + { + "properties": { + "email": { + "type": "string" + }, + "id": { + "type": "string" + }, + "type": { + "enum": [ + "email" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "phone_number": { + "type": "string" + }, + "type": { + "enum": [ + "phone_number" + ] + } + }, + "type": "object" + }, + { + "properties": { + "created_time": { + "type": "string" + }, + "id": { + "type": "string" + }, + "type": { + "enum": [ + "created_time" + ] + } + }, + "type": "object" + }, + { + "properties": { + "created_by": { + "$ref": "#/$defs/user.json" + }, + "id": { + "type": "string" + }, + "type": { + "enum": [ + "created_by" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "last_edited_time": { + "type": "string" + }, + "type": { + "enum": [ + "last_edited_time" + ] + } + }, + "type": "object" + }, + { + "properties": { + "id": { + "type": "string" + }, + "last_edited_by": { + "$ref": "#/$defs/user.json" + }, + "type": { + "enum": [ + "last_edited_by" + ] + } + }, + "type": "object" + } + ], + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "key": [ + "/id" + ] + }, + { + "recommendedName": "blocks", + "resourceConfig": { + "stream": "blocks", + "syncMode": "incremental", + "cursorField": [ + "last_edited_time" + ] + }, + "documentSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "archived": { + "type": "boolean" + }, + "bookmark": { + "properties": { + "caption": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "bulleted_list_item": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "children": { + "items": { + "type": "object" + }, + "type": "array" + }, + "color": { + "type": "string" + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "callout": { + "properties": { + "children": { + "items": { + "type": "object" + }, + "type": "array" + }, + "color": { + "type": "string" + }, + "icon": { + "$schema": "http://json-schema.org/draft-07/schema#", + "anyOf": [ + { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "caption": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "external": { + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "file": { + "additionalProperties": true, + "properties": { + "expiry_time": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "enum": [ + "file", + "external" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "emoji": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + } + ] + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "child_database": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "title": { + "type": "string" + } + }, + "type": "object" + }, + "child_page": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "title": { + "type": "string" + } + }, + "type": "object" + }, + "code": { + "properties": { + "color": { + "type": "string" + }, + "language": { + "type": "string" + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "created_time": { + "type": "string" + }, + "embed": { + "properties": { + "url": { + "type": "string" + } + }, + "type": "object" + }, + "equation": { + "properties": { + "expression": { + "type": "string" + } + }, + "type": "object" + }, + "file": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "caption": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "external": { + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "file": { + "additionalProperties": true, + "properties": { + "expiry_time": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "has_children": { + "type": [ + "null", + "boolean" + ] + }, + "heading_1": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "color": { + "type": "string" + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": [ + "null", + "object" + ] + }, + "heading_2": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "color": { + "type": "string" + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": [ + "null", + "object" + ] + }, + "heading_3": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "color": { + "type": "string" + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": [ + "null", + "object" + ] + }, + "id": { + "type": "string" + }, + "image": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "caption": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "external": { + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "file": { + "additionalProperties": true, + "properties": { + "expiry_time": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "last_edited_time": { + "type": "string" + }, + "numbered_list_item": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "children": { + "items": { + "type": "object" + }, + "type": "array" + }, + "color": { + "type": "string" + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "object": { + "type": "string" + }, + "paragraph": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "children": { + "items": { + "type": "object" + }, + "type": "array" + }, + "color": { + "type": "string" + }, + "rich_text": { + "items": { + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "pdf": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "caption": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "external": { + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "file": { + "additionalProperties": true, + "properties": { + "expiry_time": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "quote": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "children": { + "items": { + "type": "object" + }, + "type": "array" + }, + "color": { + "type": "string" + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "to_do": { + "properties": { + "checked": { + "type": [ + "null", + "boolean" + ] + }, + "children": { + "items": { + "type": "object" + }, + "type": "array" + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "toggle": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "children": { + "items": { + "type": "object" + }, + "type": "array" + }, + "color": { + "type": "string" + }, + "text": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "type": { + "type": "string" + }, + "vidoe": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "caption": { + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "annotations": { + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "rich_text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "text": { + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "type": [ + "null", + "array" + ] + }, + "external": { + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "file": { + "additionalProperties": true, + "properties": { + "expiry_time": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + }, + "type": { + "type": "string" + } + }, + "type": [ + "null", + "object" + ] + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "key": [ + "/id" + ] + } +] diff --git a/source-notion/tests/snapshots/snapshots__spec__capture.stdout.json b/source-notion/tests/snapshots/snapshots__spec__capture.stdout.json new file mode 100644 index 0000000000..b7ab8f78b8 --- /dev/null +++ b/source-notion/tests/snapshots/snapshots__spec__capture.stdout.json @@ -0,0 +1,148 @@ +[ + { + "protocol": 3032023, + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "credentials": { + "description": "Pick an authentication method.", + "discriminator": { + "propertyName": "auth_type" + }, + "oneOf": [ + { + "properties": { + "access_token": { + "airbyte_secret": true, + "description": "Access Token is a token you received by complete the OauthWebFlow of Notion.", + "title": "Access Token", + "type": "string" + }, + "auth_type": { + "const": "OAuth2.0", + "default": "OAuth2.0", + "type": "string" + }, + "client_id": { + "airbyte_secret": true, + "description": "The ClientID of your Notion integration.", + "title": "Client ID", + "type": "string" + }, + "client_secret": { + "airbyte_secret": true, + "description": "The ClientSecret of your Notion integration.", + "title": "Client Secret", + "type": "string" + } + }, + "required": [ + "auth_type", + "client_id", + "client_secret", + "access_token" + ], + "title": "OAuth2.0", + "type": "object", + "x-oauth2-provider": "notion" + }, + { + "properties": { + "auth_type": { + "const": "token", + "default": "token", + "type": "string" + }, + "token": { + "airbyte_secret": true, + "description": "Notion API access token, see the docs for more information: https://go.estuary.dev/u5BKFR", + "title": "Access Token", + "type": "string" + } + }, + "required": [ + "auth_type", + "token" + ], + "title": "Access Token", + "type": "object" + } + ], + "order": 1, + "title": "Authenticate using", + "type": "object" + }, + "start_date": { + "default": "2023-04-17T00:00:00.000Z", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000Z. Any data before this date will not be replicated.", + "examples": [ + "2020-11-16T00:00:00.000Z" + ], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", + "title": "Start Date", + "type": "string" + } + }, + "required": [ + "start_date", + "credentials" + ], + "title": "Notion Source Spec", + "type": "object" + }, + "resourceConfigSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ResourceSpec", + "type": "object", + "required": [ + "stream", + "syncMode" + ], + "properties": { + "cursorField": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "stream": { + "type": "string" + }, + "syncMode": { + "type": "string", + "enum": [ + "incremental", + "full_refresh" + ] + } + } + }, + "documentationUrl": "https://go.estuary.dev/9iXgpB", + "oauth2": { + "provider": "notion", + "authUrlTemplate": "https://api.notion.com/v1/oauth/authorize?client_id={{#urlencode}}{{{ client_id }}}{{/urlencode}}&redirect_uri={{#urlencode}}{{{ redirect_uri }}}{{/urlencode}}&response_type=code&owner=user&state={{#urlencode}}{{{ state }}}{{/urlencode}}", + "accessTokenUrlTemplate": "https://api.notion.com/v1/oauth/token", + "accessTokenBody": "{\"grant_type\": \"authorization_code\", \"redirect_uri\": \"{{{ redirect_uri }}}\", \"code\": \"{{{ code }}}\"}", + "accessTokenHeaders": { + "Authorization": "Basic {{#basicauth}}{{{ client_id }}}:{{{client_secret }}}{{/basicauth}}", + "Content-Type": "application/json" + }, + "accessTokenResponseMap": { + "access_token": "/access_token" + } + }, + "resourcePathPointers": [ + "/namespace", + "/stream" + ] + } +] diff --git a/source-notion/tests/test_snapshots.py b/source-notion/tests/test_snapshots.py new file mode 100644 index 0000000000..c3031e5192 --- /dev/null +++ b/source-notion/tests/test_snapshots.py @@ -0,0 +1,62 @@ +import json +import subprocess + + +def test_capture(request, snapshot): + result = subprocess.run( + [ + "flowctl", + "preview", + "--source", + request.fspath.dirname + "/../test.flow.yaml", + "--sessions", + "1", + "--delay", + "10s", + ], + stdout=subprocess.PIPE, + text=True, + ) + assert result.returncode == 0 + lines = [json.loads(l) for l in result.stdout.splitlines()[:50]] + + assert snapshot("capture.stdout.json") == lines + + +def test_discover(request, snapshot): + result = subprocess.run( + [ + "flowctl", + "raw", + "discover", + "--source", + request.fspath.dirname + "/../test.flow.yaml", + "-o", + "json", + "--emit-raw", + ], + stdout=subprocess.PIPE, + text=True, + ) + assert result.returncode == 0 + lines = [json.loads(l) for l in result.stdout.splitlines()] + + assert snapshot("capture.stdout.json") == lines + + +def test_spec(request, snapshot): + result = subprocess.run( + [ + "flowctl", + "raw", + "spec", + "--source", + request.fspath.dirname + "/../test.flow.yaml", + ], + stdout=subprocess.PIPE, + text=True, + ) + assert result.returncode == 0 + lines = [json.loads(l) for l in result.stdout.splitlines()] + + assert snapshot("capture.stdout.json") == lines From 5bf9897afd4748162757a811331531e4075e187f Mon Sep 17 00:00:00 2001 From: Will Baker Date: Fri, 22 Mar 2024 13:07:12 -0400 Subject: [PATCH 51/96] source-notion: snapshot tests for imported connector The new imported connector supports the new binding `comments`. Also, removed the huge lists of schematized items from the arrays in `database` and `pages`, since these were not accurate and it does not seem practical to keep them up to date and correct. --- source-notion/acmeCo/blocks.schema.yaml | 2685 +++--- source-notion/acmeCo/comments.schema.yaml | 159 + source-notion/acmeCo/databases.schema.yaml | 1023 +-- source-notion/acmeCo/flow.yaml | 4 + source-notion/acmeCo/pages.schema.yaml | 912 +- source-notion/acmeCo/users.schema.yaml | 95 +- source-notion/source_notion/__main__.py | 46 + .../source_notion/schemas/databases.json | 494 +- .../source_notion/schemas/pages.json | 284 +- .../source_notion/schemas/shared/user.json | 43 +- source-notion/source_notion/spec.json | 16 +- source-notion/test.flow.yaml | 15 +- .../snapshots__capture__capture.stdout.json | 68 +- .../snapshots__discover__capture.stdout.json | 7313 ++++++++++------- .../snapshots__spec__capture.stdout.json | 174 +- 15 files changed, 7477 insertions(+), 5854 deletions(-) create mode 100644 source-notion/acmeCo/comments.schema.yaml create mode 100644 source-notion/source_notion/__main__.py diff --git a/source-notion/acmeCo/blocks.schema.yaml b/source-notion/acmeCo/blocks.schema.yaml index 11509b8a0f..eaee72746b 100644 --- a/source-notion/acmeCo/blocks.schema.yaml +++ b/source-notion/acmeCo/blocks.schema.yaml @@ -1,54 +1,267 @@ --- $schema: "http://json-schema.org/draft-07/schema#" +type: object additionalProperties: true properties: + object: + enum: + - block + parent: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + type: + enum: + - block_id + - database_id + - page_id + - workspace + block_id: + type: string + database_id: + type: string + page_id: + type: string + workspace: + type: boolean + id: + type: string + created_time: + type: string + format: date-time + created_by: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + object: + enum: + - user + id: + type: string + name: + type: + - "null" + - string + avatar_url: + type: + - "null" + - string + type: + enum: + - person + - bot + person: + type: + - "null" + - object + additionalProperties: true + properties: + email: + type: + - "null" + - string + bot: + type: + - "null" + - object + additionalProperties: true + properties: + owner: + type: object + properties: + type: + type: string + info: + type: + - object + - boolean + properties: + avatar_url: + type: + - "null" + - string + id: + type: + - "null" + - string + name: + type: + - "null" + - string + object: + type: + - "null" + - string + person: + type: + - "null" + - object + properties: + email: + type: + - "null" + - string + type: + type: + - "null" + - string + workspace: + type: + - "null" + - boolean + workspace_name: + type: + - "null" + - string + last_edited_by: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + object: + enum: + - user + id: + type: string + name: + type: + - "null" + - string + avatar_url: + type: + - "null" + - string + type: + enum: + - person + - bot + person: + type: + - "null" + - object + additionalProperties: true + properties: + email: + type: + - "null" + - string + bot: + type: + - "null" + - object + additionalProperties: true + properties: + owner: + type: object + properties: + type: + type: string + info: + type: + - object + - boolean + properties: + avatar_url: + type: + - "null" + - string + id: + type: + - "null" + - string + name: + type: + - "null" + - string + object: + type: + - "null" + - string + person: + type: + - "null" + - object + properties: + email: + type: + - "null" + - string + type: + type: + - "null" + - string + workspace: + type: + - "null" + - boolean + workspace_name: + type: + - "null" + - string + last_edited_time: + type: string + format: date-time archived: type: boolean + has_children: + type: + - "null" + - boolean + type: + enum: + - bookmark + - breadcrumb + - bulleted_list_item + - callout + - child_database + - child_page + - code + - column + - column_list + - divider + - embed + - equation + - file + - heading_1 + - heading_2 + - heading_3 + - image + - link_preview + - link_to_page + - numbered_list_item + - paragraph + - pdf + - quote + - synced_block + - table + - table_of_contents + - table_row + - template + - to_do + - toggle + - unsupported + - video bookmark: + type: object properties: + url: + type: string caption: + type: array items: $schema: "http://json-schema.org/draft-07/schema#" + type: object properties: - annotations: - additionalProperties: true - properties: - bold: - type: - - "null" - - boolean - code: - type: - - "null" - - boolean - color: - type: - - "null" - - string - italic: - type: - - "null" - - boolean - strikethrough: - type: - - "null" - - boolean - underline: - type: - - "null" - - boolean - type: - - "null" - - object - href: + type: type: - "null" - string - plain_text: + text: type: - "null" - - string - rich_text: + - object additionalProperties: true properties: content: @@ -56,104 +269,112 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - url: - type: string - type: object - bulleted_list_item: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - children: - items: - type: object - type: array - color: - type: string - text: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + breadcrumb: + type: object + bulleted_list_item: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + color: + type: string + rich_text: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -161,106 +382,231 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: object - callout: - properties: + - object + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + href: + type: + - "null" + - string children: + type: array items: type: object - type: array + callout: + type: object + properties: color: type: string + rich_text: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: + type: + - "null" + - string + text: + type: + - "null" + - object + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + type: + - "null" + - object + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + mention: + type: + - "null" + - object + additionalProperties: true + properties: + type: + type: + - "null" + - string + info: + type: + - "null" + - object + properties: + id: + type: + - "null" + - string + object: + type: + - "null" + - string + equation: + type: + - "null" + - object + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: + type: + - "null" + - object + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + href: + type: + - "null" + - string icon: $schema: "http://json-schema.org/draft-07/schema#" anyOf: - $schema: "http://json-schema.org/draft-07/schema#" + type: + - "null" + - object additionalProperties: true properties: + type: + enum: + - file + - external caption: + type: + - "null" + - array items: $schema: "http://json-schema.org/draft-07/schema#" + type: object properties: - annotations: - additionalProperties: true - properties: - bold: - type: - - "null" - - boolean - code: - type: - - "null" - - boolean - color: - type: - - "null" - - string - italic: - type: - - "null" - - boolean - strikethrough: - type: - - "null" - - boolean - underline: - type: - - "null" - - boolean - type: - - "null" - - object - href: + type: type: - "null" - string - plain_text: + text: type: - "null" - - string - rich_text: + - object additionalProperties: true properties: content: @@ -268,6 +614,9 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: @@ -277,121 +626,241 @@ properties: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - enum: - - url - url: + id: + type: + - "null" + - string + object: type: - "null" - string + equation: + type: + - "null" + - object + additionalProperties: true + properties: + expression: type: - "null" - - object + - string + annotations: type: - "null" - object + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + href: + type: + - "null" + - string + external: + type: + - "null" + - object + additionalProperties: true + properties: + url: + type: string + file: + type: + - "null" + - object + additionalProperties: true + properties: + url: + type: string + expiry_time: + type: string + format: date-time + - $schema: "http://json-schema.org/draft-07/schema#" + type: + - "null" + - object + additionalProperties: true + properties: + type: + type: string + emoji: + type: string + child_page: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + title: + type: string + child_database: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + title: + type: string + code: + type: object + properties: + caption: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: + type: + - "null" + - string + text: + type: + - "null" + - object + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + type: + - "null" + - object + additionalProperties: true + properties: type: + enum: + - url + url: + type: + - "null" + - string + mention: + type: + - "null" + - object + additionalProperties: true + properties: + type: + type: + - "null" + - string + info: + type: + - "null" + - object + properties: + id: + type: + - "null" + - string + object: type: - "null" - string - type: object - type: - - "null" - - array - external: - additionalProperties: true - properties: - url: - type: string - type: - - "null" - - object - file: - additionalProperties: true - properties: - expiry_time: - type: string - url: - type: string - type: - - "null" - - object - type: - enum: - - file - - external - type: - - "null" - - object - - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - emoji: - type: string + equation: type: - type: string - type: - - "null" - - object - text: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: + - "null" + - object + additionalProperties: true + properties: + expression: + type: + - "null" + - string annotations: + type: + - "null" + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + rich_text: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -399,112 +868,206 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: object - child_database: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - title: - type: string - type: object - child_page: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - title: - type: string - type: object - code: - properties: - color: - type: string - language: - type: string - text: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + language: + enum: + - abap + - arduino + - bash + - basic + - c + - clojure + - coffeescript + - c++ + - "c#" + - css + - dart + - diff + - docker + - elixir + - elm + - erlang + - flow + - fortran + - "f#" + - gherkin + - glsl + - go + - graphql + - groovy + - haskell + - html + - java + - javascript + - json + - julia + - kotlin + - latex + - less + - lisp + - livescript + - lua + - makefile + - markdown + - markup + - matlab + - mermaid + - nix + - objective-c + - ocaml + - pascal + - perl + - php + - plain text + - powershell + - prolog + - protobuf + - python + - r + - reason + - ruby + - rust + - sass + - scala + - scheme + - scss + - shell + - sql + - swift + - typescript + - vb.net + - verilog + - vhdl + - visual basic + - webassembly + - xml + - yaml + - "java/c/c++/c#" + column: + type: object + column_list: + type: object + divider: + type: object + embed: + type: object + properties: + url: + type: string + equation: + type: object + properties: + expression: + type: string + file: + $schema: "http://json-schema.org/draft-07/schema#" + type: + - "null" + - object + additionalProperties: true + properties: + type: + enum: + - file + - external + caption: + type: + - "null" + - array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -512,84 +1075,61 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: object - created_time: - type: string - embed: - properties: - url: - type: string - type: object - equation: - properties: - expression: - type: string - type: object - file: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - caption: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: - type: - - "null" - - boolean - color: - type: - - "null" - - string italic: type: - "null" @@ -602,145 +1142,64 @@ properties: type: - "null" - boolean - type: - - "null" - - object - href: - type: - - "null" - - string - plain_text: - type: - - "null" - - string - rich_text: - additionalProperties: true - properties: - content: - type: - - "null" - - string - link: - additionalProperties: true - properties: - type: - type: string - url: - type: - - "null" - - string + code: type: - "null" - - object - type: - - "null" - - object - text: - additionalProperties: true - properties: - content: + - boolean + color: type: - "null" - string - link: - additionalProperties: true - properties: - type: - type: string - url: - type: - - "null" - - string - type: - - "null" - - object + plain_text: type: - "null" - - object - type: + - string + href: type: - "null" - string - type: object + external: type: - "null" - - array - external: + - object additionalProperties: true properties: url: type: string + file: type: - "null" - object - file: additionalProperties: true properties: - expiry_time: - type: string url: type: string - type: - - "null" - - object - type: - type: string + expiry_time: + type: string + format: date-time + heading_1: + $schema: "http://json-schema.org/draft-07/schema#" type: - "null" - object - has_children: - type: - - "null" - - boolean - heading_1: - $schema: "http://json-schema.org/draft-07/schema#" additionalProperties: true properties: color: type: string - text: + rich_text: + type: array items: $schema: "http://json-schema.org/draft-07/schema#" + type: object properties: - annotations: - additionalProperties: true - properties: - bold: - type: - - "null" - - boolean - code: - type: - - "null" - - boolean - color: - type: - - "null" - - string - italic: - type: - - "null" - - boolean - strikethrough: - type: - - "null" - - boolean - underline: - type: - - "null" - - boolean - type: - - "null" - - object - href: + type: type: - "null" - string - plain_text: + text: type: - "null" - - string - rich_text: + - object additionalProperties: true properties: content: @@ -748,100 +1207,114 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: - - "null" - - object - heading_2: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - color: - type: string - text: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + is_toggleable: + type: boolean + heading_2: + $schema: "http://json-schema.org/draft-07/schema#" + type: + - "null" + - object + additionalProperties: true + properties: + color: + type: string + rich_text: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -849,100 +1322,114 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: - - "null" - - object - heading_3: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - color: - type: string - text: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + is_toggleable: + type: boolean + heading_3: + $schema: "http://json-schema.org/draft-07/schema#" + type: + - "null" + - object + additionalProperties: true + properties: + color: + type: string + rich_text: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -950,100 +1437,118 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: - - "null" - - object - id: - type: string - image: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - caption: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + is_toggleable: + type: boolean + image: + $schema: "http://json-schema.org/draft-07/schema#" + type: + - "null" + - object + additionalProperties: true + properties: + type: + enum: + - file + - external + caption: + type: + - "null" + - array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -1051,128 +1556,141 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: + type: + - "null" + - string + object: type: - "null" - string + equation: + type: + - "null" + - object + additionalProperties: true + properties: + expression: type: - "null" - - object + - string + annotations: type: - "null" - object - type: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + plain_text: type: - "null" - string - type: object + href: + type: + - "null" + - string + external: type: - "null" - - array - external: + - object additionalProperties: true properties: url: type: string + file: type: - "null" - object - file: additionalProperties: true properties: - expiry_time: - type: string url: type: string - type: - - "null" - - object + expiry_time: + type: string + format: date-time + link_preview: + type: object + properties: + url: + type: string + link_to_page: + type: object + properties: + page_id: + type: string type: type: string - type: - - "null" - - object - last_edited_time: - type: string numbered_list_item: $schema: "http://json-schema.org/draft-07/schema#" + type: object additionalProperties: true properties: - children: - items: - type: object - type: array color: type: string - text: + rich_text: + type: array items: $schema: "http://json-schema.org/draft-07/schema#" + type: object properties: - annotations: - additionalProperties: true - properties: - bold: - type: - - "null" - - boolean - code: - type: - - "null" - - boolean - color: - type: - - "null" - - string - italic: - type: - - "null" - - boolean - strikethrough: - type: - - "null" - - boolean - underline: - type: - - "null" - - boolean - type: - - "null" - - object - href: + type: type: - "null" - string - plain_text: + text: type: - "null" - - string - rich_text: + - object additionalProperties: true properties: content: @@ -1180,103 +1698,114 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: object - object: - type: string - paragraph: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - children: - items: - type: object - type: array - color: - type: string - rich_text: - items: - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + children: + type: array + items: + type: object + paragraph: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + color: + type: string + rich_text: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -1284,91 +1813,120 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - text: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + children: + type: array + items: + type: object + pdf: + $schema: "http://json-schema.org/draft-07/schema#" + type: + - "null" + - object + additionalProperties: true + properties: + type: + enum: + - file + - external + caption: + type: + - "null" + - array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -1376,96 +1934,129 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: object - pdf: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - caption: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + external: + type: + - "null" + - object + additionalProperties: true + properties: + url: + type: string + file: + type: + - "null" + - object + additionalProperties: true + properties: + url: + type: string + expiry_time: + type: string + format: date-time + quote: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + color: + type: string + rich_text: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -1473,126 +2064,260 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: - - "null" - - array - external: - additionalProperties: true - properties: - url: - type: string - type: - - "null" - - object - file: - additionalProperties: true - properties: - expiry_time: - type: string - url: - type: string - type: - - "null" - - object - type: - type: string - type: - - "null" - - object - quote: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - children: - items: - type: object - type: array - color: - type: string - text: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + children: + type: array + items: + type: object + synced_block: + type: object + properties: + synced_from: + type: + - "null" + - object + properties: + type: + type: string + enum: + - block_id + block_id: + type: string + children: + type: + - "null" + - array + items: + type: + - "null" + - object + additionalProperties: true + table: + type: object + properties: + table_width: + type: integer + has_column_header: + type: boolean + has_row_header: + type: boolean + table_of_contents: + type: object + properties: + color: + type: string + table_row: + type: object + properties: + cells: + type: + - "null" + - array + items: + type: + - "null" + - array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: + type: + - "null" + - string + text: + type: + - "null" + - object + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + type: + - "null" + - object + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + mention: + type: + - "null" + - object + additionalProperties: true + properties: + type: + type: + - "null" + - string + info: + type: + - "null" + - object + properties: + id: + type: + - "null" + - string + object: + type: + - "null" + - string + equation: + type: + - "null" + - object + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: + type: + - "null" + - object + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + href: + type: + - "null" + - string + template: + type: object + properties: + rich_text: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -1600,102 +2325,106 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: object - to_do: - properties: - checked: - type: - - "null" - - boolean - children: - items: - type: object - type: array - text: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + to_do: + type: object + properties: + rich_text: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -1703,102 +2432,120 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: object - toggle: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - children: - items: - type: object - type: array - color: - type: string - text: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + checked: + type: + - "null" + - boolean + color: + type: string + children: + type: array + items: + type: object + toggle: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + color: + type: string + rich_text: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -1806,98 +2553,120 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: type: - "null" - string - type: - - "null" - - object + object: + type: + - "null" + - string + equation: type: - "null" - object - type: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - type: object - type: array - type: object - type: - type: string - vidoe: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - caption: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: bold: type: - "null" - boolean - code: + italic: type: - "null" - boolean - color: + strikethrough: type: - "null" - - string - italic: + - boolean + underline: type: - "null" - boolean - strikethrough: + code: type: - "null" - boolean - underline: + color: type: - "null" - - boolean + - string + plain_text: type: - "null" - - object + - string href: type: - "null" - string - plain_text: + children: + type: array + items: + type: object + video: + $schema: "http://json-schema.org/draft-07/schema#" + type: + - "null" + - object + additionalProperties: true + properties: + type: + enum: + - file + - external + caption: + type: + - "null" + - array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: type: - "null" - string - rich_text: + text: + type: + - "null" + - object additionalProperties: true properties: content: @@ -1905,73 +2674,115 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: + type: + - "null" + - string + object: type: - "null" - string + equation: + type: + - "null" + - object + additionalProperties: true + properties: + expression: type: - "null" - - object + - string + annotations: type: - "null" - object - type: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + plain_text: type: - "null" - string - type: object + href: + type: + - "null" + - string + external: type: - "null" - - array - external: + - object additionalProperties: true properties: url: type: string + file: type: - "null" - object - file: additionalProperties: true properties: - expiry_time: - type: string url: type: string - type: - - "null" - - object - type: - type: string - type: - - "null" - - object -required: - - id -type: object + expiry_time: + type: string + format: date-time + unsupported: + type: object + _meta: + type: object + properties: + row_id: + type: integer + required: + - row_id +x-infer-schema: true diff --git a/source-notion/acmeCo/comments.schema.yaml b/source-notion/acmeCo/comments.schema.yaml new file mode 100644 index 0000000000..53532ada3e --- /dev/null +++ b/source-notion/acmeCo/comments.schema.yaml @@ -0,0 +1,159 @@ +--- +$schema: "http://json-schema.org/draft-07/schema#" +type: object +additionalProperties: true +properties: + object: + enum: + - comment + id: + type: string + parent: + type: object + properties: + type: + enum: + - page_id + page_id: + type: string + discussion_id: + type: string + created_time: + type: string + format: date-time + last_edited_time: + type: string + format: date-time + page_last_edited_time: + type: string + format: date-time + created_by: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + object: + enum: + - user + id: + type: string + name: + type: + - "null" + - string + avatar_url: + type: + - "null" + - string + type: + enum: + - person + - bot + person: + type: + - "null" + - object + additionalProperties: true + properties: + email: + type: + - "null" + - string + bot: + type: + - "null" + - object + additionalProperties: true + properties: + owner: + type: object + properties: + type: + type: string + info: + type: + - object + - boolean + properties: + avatar_url: + type: + - "null" + - string + id: + type: + - "null" + - string + name: + type: + - "null" + - string + object: + type: + - "null" + - string + person: + type: + - "null" + - object + properties: + email: + type: + - "null" + - string + type: + type: + - "null" + - string + workspace: + type: + - "null" + - boolean + workspace_name: + type: + - "null" + - string + rich_text: + type: array + items: + type: object + properties: + type: + type: string + text: + type: object + properties: + content: + type: string + link: + type: + - "null" + - object + annotations: + type: object + properties: + bold: + type: boolean + italic: + type: boolean + strikethrough: + type: boolean + underline: + type: boolean + code: + type: boolean + color: + type: string + plain_text: + type: string + href: + type: + - "null" + - string + _meta: + type: object + properties: + row_id: + type: integer + required: + - row_id +x-infer-schema: true diff --git a/source-notion/acmeCo/databases.schema.yaml b/source-notion/acmeCo/databases.schema.yaml index 0a1d69ef37..9514d9c631 100644 --- a/source-notion/acmeCo/databases.schema.yaml +++ b/source-notion/acmeCo/databases.schema.yaml @@ -1,245 +1,428 @@ --- -$defs: - options.json: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - color: - enum: - - default - - gray - - brown - - orange - - yellow - - green - - blue - - purple - - pink - - red - id: - type: string - name: - type: string - type: - - "null" - - object $schema: "http://json-schema.org/draft-07/schema#" +type: object additionalProperties: true properties: - archived: - type: boolean - cover: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - caption: - items: - $schema: "http://json-schema.org/draft-07/schema#" + object: + enum: + - database + id: + type: string + created_time: + type: string + format: date-time + last_edited_time: + type: string + format: date-time + title: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: + type: + - "null" + - string + text: + type: + - "null" + - object + additionalProperties: true properties: - annotations: + content: + type: + - "null" + - string + link: + type: + - "null" + - object additionalProperties: true properties: - bold: - type: - - "null" - - boolean - code: - type: - - "null" - - boolean - color: + type: + enum: + - url + url: type: - "null" - string - italic: - type: - - "null" - - boolean - strikethrough: + mention: + type: + - "null" + - object + additionalProperties: true + properties: + type: + type: + - "null" + - string + info: + type: + - "null" + - object + properties: + id: type: - "null" - - boolean - underline: + - string + object: type: - "null" - - boolean + - string + equation: + type: + - "null" + - object + additionalProperties: true + properties: + expression: type: - "null" - - object - href: + - string + annotations: + type: + - "null" + - object + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: type: - "null" - string - plain_text: + plain_text: + type: + - "null" + - string + href: + type: + - "null" + - string + description: + type: array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + properties: + type: + type: + - "null" + - string + text: + type: + - "null" + - object + additionalProperties: true + properties: + content: type: - "null" - string - rich_text: + link: + type: + - "null" + - object additionalProperties: true properties: - content: + type: + enum: + - url + url: type: - "null" - string - link: - additionalProperties: true - properties: - type: - type: string - url: - type: - - "null" - - string - type: - - "null" - - object + mention: + type: + - "null" + - object + additionalProperties: true + properties: + type: + type: + - "null" + - string + info: type: - "null" - object - text: - additionalProperties: true properties: - content: + id: type: - "null" - string - link: - additionalProperties: true - properties: - type: - type: string - url: - type: - - "null" - - string + object: type: - "null" - - object + - string + equation: + type: + - "null" + - object + additionalProperties: true + properties: + expression: type: - "null" - - object - type: + - string + annotations: + type: + - "null" + - object + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: type: - "null" - string - type: object + plain_text: + type: + - "null" + - string + href: + type: + - "null" + - string + last_edited_by: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + object: + enum: + - user + id: + type: string + name: type: - "null" - - array - external: - additionalProperties: true - properties: - url: - type: string + - string + avatar_url: type: - "null" - - object - file: - additionalProperties: true - properties: - expiry_time: - type: string - url: - type: string + - string + type: + enum: + - person + - bot + person: type: - "null" - object - type: - type: string - type: - - "null" - - object - created_by: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - avatar_url: + additionalProperties: true + properties: + email: + type: + - "null" + - string + bot: type: - "null" - - string - bot: + - object additionalProperties: true properties: owner: + type: object properties: type: type: string + info: + type: + - object + - boolean + properties: + avatar_url: + type: + - "null" + - string + id: + type: + - "null" + - string + name: + type: + - "null" + - string + object: + type: + - "null" + - string + person: + type: + - "null" + - object + properties: + email: + type: + - "null" + - string + type: + type: + - "null" + - string workspace: type: - "null" - boolean - type: object - type: - - "null" - - object + workspace_name: + type: + - "null" + - string + created_by: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + object: + enum: + - user id: type: string name: - type: string - object: - type: string + type: + - "null" + - string + avatar_url: + type: + - "null" + - string + type: + enum: + - person + - bot person: + type: + - "null" + - object additionalProperties: true properties: email: - type: string + type: + - "null" + - string + bot: type: - "null" - object - type: - type: string - type: object - created_time: - type: string + additionalProperties: true + properties: + owner: + type: object + properties: + type: + type: string + info: + type: + - object + - boolean + properties: + avatar_url: + type: + - "null" + - string + id: + type: + - "null" + - string + name: + type: + - "null" + - string + object: + type: + - "null" + - string + person: + type: + - "null" + - object + properties: + email: + type: + - "null" + - string + type: + type: + - "null" + - string + workspace: + type: + - "null" + - boolean + workspace_name: + type: + - "null" + - string + archived: + type: boolean icon: $schema: "http://json-schema.org/draft-07/schema#" anyOf: - $schema: "http://json-schema.org/draft-07/schema#" + type: + - "null" + - object additionalProperties: true properties: + type: + enum: + - file + - external caption: + type: + - "null" + - array items: $schema: "http://json-schema.org/draft-07/schema#" + type: object properties: - annotations: - additionalProperties: true - properties: - bold: - type: - - "null" - - boolean - code: - type: - - "null" - - boolean - color: - type: - - "null" - - string - italic: - type: - - "null" - - boolean - strikethrough: - type: - - "null" - - boolean - underline: - type: - - "null" - - boolean - type: - - "null" - - object - href: + type: type: - "null" - string - plain_text: + text: type: - "null" - - string - rich_text: + - object additionalProperties: true properties: content: @@ -247,6 +430,9 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: @@ -256,403 +442,278 @@ properties: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - enum: - - url - url: + id: + type: + - "null" + - string + object: type: - "null" - string + equation: + type: + - "null" + - object + additionalProperties: true + properties: + expression: type: - "null" - - object + - string + annotations: type: - "null" - object - type: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + plain_text: type: - "null" - string - type: object + href: + type: + - "null" + - string + external: type: - "null" - - array - external: + - object additionalProperties: true properties: url: type: string + file: type: - "null" - object - file: additionalProperties: true properties: - expiry_time: - type: string url: type: string - type: - - "null" - - object - type: - enum: - - file - - external + expiry_time: + type: string + format: date-time + - $schema: "http://json-schema.org/draft-07/schema#" type: - "null" - object - - $schema: "http://json-schema.org/draft-07/schema#" additionalProperties: true properties: - emoji: - type: string type: type: string - type: - - "null" - - object - id: - type: string - is_inline: + emoji: + type: string + cover: + $schema: "http://json-schema.org/draft-07/schema#" type: - "null" - - boolean - last_edited_by: - $schema: "http://json-schema.org/draft-07/schema#" + - object additionalProperties: true properties: - avatar_url: + type: + enum: + - file + - external + caption: type: - "null" - - string - bot: - additionalProperties: true - properties: - owner: - properties: - type: - type: string - workspace: - type: - - "null" - - boolean - type: object - type: - - "null" - - object - id: - type: string - name: - type: string - object: - type: string - person: - additionalProperties: true - properties: - email: - type: string - type: - - "null" - - object - type: - type: string - type: object - last_edited_time: - type: string - object: - type: string - parent: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - database_id: - type: string - page_id: - type: string - type: - type: string - workspace: - type: boolean - type: object - properties: - items: - additionalProperties: true - properties: - name: - type: - - "null" - - string - value: - additionalProperties: true - anyOf: - - properties: - id: - type: string - type: - type: string - type: object - - additionalProperties: true - properties: - id: - type: string - name: - type: string - type: - enum: - - title - - rich_text - - date - - people - - files - - checkbox - - url - - email - - phone_number - - created_time - - created_by - - last_edited_time - - last_edited_by - type: object - - properties: - format: - enum: - - number - - number_with_commas - - percent - - dollar - - canadian_dollar - - euro - - pound - - yen - - ruble - - rupee - - won - - yuan - - real - - lira - - rupiah - - franc - - hong_kong_dollar - - new_zealand_dollar - - krona - - norwegian_krone - - mexican_peso - - rand - - new_taiwan_dollar - - danish_krone - - zloty - - baht - - forint - - koruna - - shekel - - chilean_peso - - philippine_peso - - dirham - - colombian_peso - - riyal - - ringgit - - leu - id: - type: string - name: - type: string - type: - enum: - - number - type: object - - properties: - id: - type: string - name: - type: string - options: - items: - $ref: "#/$defs/options.json" - type: array - type: - enum: - - select - - multi_select - type: object - - properties: - expression: - type: string - id: - type: string - name: - type: string - type: - enum: - - formula - type: object - - properties: - database_id: - type: string - id: - type: string - name: - type: string - synced_property_id: - type: - - "null" - - string - synced_property_name: - type: - - "null" - - string - type: - enum: - - relation - type: object - - properties: - function: - enum: - - count_all - - count_values - - count_unique_values - - count_empty - - count_not_empty - - percent_empty - - percent_not_empty - - sum - - average - - median - - min - - max - - range - - show_original - id: - type: string - name: - type: string - relation_property_id: - type: string - relation_property_name: - type: string - rollup_property_id: - type: string - rollup_property_name: - type: string - type: - enum: - - rollup - type: object + - array + items: + $schema: "http://json-schema.org/draft-07/schema#" type: object - type: - - "null" - - object - type: array - title: - items: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: - additionalProperties: true properties: - bold: - type: - - "null" - - boolean - code: - type: - - "null" - - boolean - color: + type: type: - "null" - string - italic: - type: - - "null" - - boolean - strikethrough: - type: - - "null" - - boolean - underline: + text: type: - "null" - - boolean - type: - - "null" - - object - href: - type: - - "null" - - string - plain_text: - type: - - "null" - - string - rich_text: - additionalProperties: true - properties: - content: + - object + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + type: + - "null" + - object + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + mention: type: - "null" - - string - link: + - object additionalProperties: true properties: type: - type: string - url: type: - "null" - string + info: + type: + - "null" + - object + properties: + id: + type: + - "null" + - string + object: + type: + - "null" + - string + equation: type: - "null" - object - type: - - "null" - - object - text: - additionalProperties: true - properties: - content: + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: type: - "null" - - string - link: + - object additionalProperties: true properties: - type: - type: string - url: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: type: - "null" - string + plain_text: type: - "null" - - object - type: - - "null" - - object + - string + href: + type: + - "null" + - string + external: type: - type: - - "null" - - string - type: object - type: array + - "null" + - object + additionalProperties: true + properties: + url: + type: string + file: + type: + - "null" + - object + additionalProperties: true + properties: + url: + type: string + expiry_time: + type: string + format: date-time + parent: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + type: + enum: + - block_id + - database_id + - page_id + - workspace + block_id: + type: string + database_id: + type: string + page_id: + type: string + workspace: + type: boolean url: type: string -required: - - id -type: object + is_inline: + type: + - "null" + - boolean + public_url: + type: + - "null" + - string + properties: + type: array + _meta: + type: object + properties: + row_id: + type: integer + required: + - row_id +x-infer-schema: true diff --git a/source-notion/acmeCo/flow.yaml b/source-notion/acmeCo/flow.yaml index 29c9f9a470..668b6c0e22 100644 --- a/source-notion/acmeCo/flow.yaml +++ b/source-notion/acmeCo/flow.yaml @@ -4,6 +4,10 @@ collections: schema: blocks.schema.yaml key: - /id + acmeCo/comments: + schema: comments.schema.yaml + key: + - /id acmeCo/databases: schema: databases.schema.yaml key: diff --git a/source-notion/acmeCo/pages.schema.yaml b/source-notion/acmeCo/pages.schema.yaml index 934550d015..e6c79d2b4a 100644 --- a/source-notion/acmeCo/pages.schema.yaml +++ b/source-notion/acmeCo/pages.schema.yaml @@ -1,308 +1,366 @@ --- -$defs: - date.json: +$schema: "http://json-schema.org/draft-07/schema#" +type: object +additionalProperties: true +properties: + object: + enum: + - page + id: + type: string + created_time: + type: string + format: date-time + created_by: $schema: "http://json-schema.org/draft-07/schema#" + type: object additionalProperties: true properties: - end: + object: + enum: + - user + id: + type: string + name: type: - "null" - string - start: + avatar_url: type: - "null" - string - time_zone: + type: + enum: + - person + - bot + person: type: - "null" - - string - type: - - "null" - - object - options.json: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - color: - enum: - - default - - gray - - brown - - orange - - yellow - - green - - blue - - purple - - pink - - red - id: - type: string - name: - type: string - type: - - "null" - - object - rich_text.json: - $schema: "http://json-schema.org/draft-07/schema#" - properties: - annotations: + - object additionalProperties: true properties: - bold: - type: - - "null" - - boolean - code: - type: - - "null" - - boolean - color: + email: type: - "null" - string - italic: - type: - - "null" - - boolean - strikethrough: - type: - - "null" - - boolean - underline: - type: - - "null" - - boolean + bot: type: - "null" - object - href: - type: - - "null" - - string - plain_text: - type: - - "null" - - string - rich_text: additionalProperties: true properties: - content: - type: - - "null" - - string - link: - additionalProperties: true + owner: + type: object properties: type: - enum: - - url - url: + type: string + info: + type: + - object + - boolean + properties: + avatar_url: + type: + - "null" + - string + id: + type: + - "null" + - string + name: + type: + - "null" + - string + object: + type: + - "null" + - string + person: + type: + - "null" + - object + properties: + email: + type: + - "null" + - string + type: + type: + - "null" + - string + workspace: type: - "null" - - string + - boolean + workspace_name: type: - "null" - - object + - string + last_edited_time: + type: string + format: date-time + last_edited_by: + $schema: "http://json-schema.org/draft-07/schema#" + type: object + additionalProperties: true + properties: + object: + enum: + - user + id: + type: string + name: + type: + - "null" + - string + avatar_url: + type: + - "null" + - string + type: + enum: + - person + - bot + person: type: - "null" - object - text: additionalProperties: true properties: - content: + email: type: - "null" - string - link: - additionalProperties: true + bot: + type: + - "null" + - object + additionalProperties: true + properties: + owner: + type: object properties: type: - enum: - - url - url: + type: string + info: + type: + - object + - boolean + properties: + avatar_url: + type: + - "null" + - string + id: + type: + - "null" + - string + name: + type: + - "null" + - string + object: + type: + - "null" + - string + person: + type: + - "null" + - object + properties: + email: + type: + - "null" + - string + type: + type: + - "null" + - string + workspace: type: - "null" - - string + - boolean + workspace_name: type: - "null" - - object + - string + archived: + type: boolean + icon: + $schema: "http://json-schema.org/draft-07/schema#" + anyOf: + - $schema: "http://json-schema.org/draft-07/schema#" type: - "null" - object - type: - type: - - "null" - - string - type: object - title.json: - items: - additionalProperties: true - properties: - annotations: - additionalProperties: true - properties: - bold: - type: - - "null" - - boolean - code: - type: - - "null" - - boolean - color: - type: - - "null" - - string - italic: - type: - - "null" - - boolean - strikethrough: - type: - - "null" - - boolean - underline: - type: - - "null" - - boolean - type: - - "null" - - object - href: - type: - - "null" - - string - plain_text: + additionalProperties: true + properties: type: - - "null" - - string - text: - properties: - content: - type: - - "null" - - string - link: - additionalProperties: true + enum: + - file + - external + caption: + type: + - "null" + - array + items: + $schema: "http://json-schema.org/draft-07/schema#" + type: object properties: type: - enum: - - url - url: type: - "null" - string - type: - - "null" - - object - type: - - "null" - - object - type: - type: - - "null" - - string - type: - - "null" - - object - type: - - "null" - - array - user.json: - $schema: "http://json-schema.org/draft-07/schema#" - additionalProperties: true - properties: - avatar_url: - type: - - "null" - - string - bot: - additionalProperties: true - properties: - owner: + text: + type: + - "null" + - object + additionalProperties: true + properties: + content: + type: + - "null" + - string + link: + type: + - "null" + - object + additionalProperties: true + properties: + type: + enum: + - url + url: + type: + - "null" + - string + mention: + type: + - "null" + - object + additionalProperties: true + properties: + type: + type: + - "null" + - string + info: + type: + - "null" + - object + properties: + id: + type: + - "null" + - string + object: + type: + - "null" + - string + equation: + type: + - "null" + - object + additionalProperties: true + properties: + expression: + type: + - "null" + - string + annotations: + type: + - "null" + - object + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + plain_text: + type: + - "null" + - string + href: + type: + - "null" + - string + external: + type: + - "null" + - object + additionalProperties: true properties: - type: + url: type: string - workspace: - type: - - "null" - - boolean - type: object + file: + type: + - "null" + - object + additionalProperties: true + properties: + url: + type: string + expiry_time: + type: string + format: date-time + - $schema: "http://json-schema.org/draft-07/schema#" type: - "null" - object - id: - type: string - name: - type: string - object: - enum: - - user - person: additionalProperties: true properties: - email: + type: + type: string + emoji: type: string - type: - - "null" - - object - type: - enum: - - person - - bot - type: object -$schema: "http://json-schema.org/draft-07/schema#" -additionalProperties: true -properties: - archived: - type: boolean cover: $schema: "http://json-schema.org/draft-07/schema#" + type: + - "null" + - object additionalProperties: true properties: + type: + enum: + - file + - external caption: + type: + - "null" + - array items: $schema: "http://json-schema.org/draft-07/schema#" + type: object properties: - annotations: - additionalProperties: true - properties: - bold: - type: - - "null" - - boolean - code: - type: - - "null" - - boolean - color: - type: - - "null" - - string - italic: - type: - - "null" - - boolean - strikethrough: - type: - - "null" - - boolean - underline: - type: - - "null" - - boolean - type: - - "null" - - object - href: + type: type: - "null" - string - plain_text: + text: type: - "null" - - string - rich_text: + - object additionalProperties: true properties: content: @@ -310,358 +368,140 @@ properties: - "null" - string link: + type: + - "null" + - object additionalProperties: true properties: type: - type: string + enum: + - url url: type: - "null" - string - type: - - "null" - - object + mention: type: - "null" - object - text: additionalProperties: true properties: - content: + type: type: - "null" - string - link: - additionalProperties: true + info: + type: + - "null" + - object properties: - type: - type: string - url: + id: + type: + - "null" + - string + object: type: - "null" - string + equation: + type: + - "null" + - object + additionalProperties: true + properties: + expression: type: - "null" - - object + - string + annotations: type: - "null" - object - type: + additionalProperties: true + properties: + bold: + type: + - "null" + - boolean + italic: + type: + - "null" + - boolean + strikethrough: + type: + - "null" + - boolean + underline: + type: + - "null" + - boolean + code: + type: + - "null" + - boolean + color: + type: + - "null" + - string + plain_text: type: - "null" - string - type: object + href: + type: + - "null" + - string + external: type: - "null" - - array - external: + - object additionalProperties: true properties: url: type: string + file: type: - "null" - object - file: additionalProperties: true properties: - expiry_time: - type: string url: type: string - type: - - "null" - - object - type: - type: string - type: - - "null" - - object - created_time: - type: string - id: - type: string - last_edited_time: - type: string - object: - type: string + expiry_time: + type: string + format: date-time parent: $schema: "http://json-schema.org/draft-07/schema#" + type: object additionalProperties: true properties: + type: + enum: + - block_id + - database_id + - page_id + - workspace + block_id: + type: string database_id: type: string page_id: type: string - type: - type: string workspace: type: boolean - type: object - properties: - items: - additionalProperties: true - properties: - name: - type: - - "null" - - string - value: - additionalProperties: true - anyOf: - - properties: - id: - type: string - type: - type: string - type: object - - properties: - id: - type: string - title: - $ref: "#/$defs/title.json" - type: - enum: - - title - type: object - - properties: - id: - type: string - rich_text: - $ref: "#/$defs/rich_text.json" - type: - enum: - - rich_text - type: object - - properties: - id: - type: string - select: - $ref: "#/$defs/options.json" - type: - enum: - - select - type: object - - properties: - id: - type: string - multi_select: - items: - $ref: "#/$defs/options.json" - type: - - "null" - - array - type: - enum: - - multi_select - type: object - - properties: - date: - $ref: "#/$defs/date.json" - id: - type: string - type: - enum: - - date - type: object - - properties: - formula: - properties: - boolean: - type: - - "null" - - boolean - date: - $ref: "#/$defs/date.json" - number: - type: - - "null" - - number - string: - type: - - "null" - - string - type: - enum: - - string - - number - - boolean - - date - type: - - "null" - - object - id: - type: string - type: - enum: - - formula - type: object - - properties: - id: - type: string - relation: - items: - properties: - id: - type: string - type: object - type: - - "null" - - array - type: - enum: - - relation - type: object - - properties: - id: - type: string - rollup: - properties: - array: - items: - properties: - type: - type: string - type: object - type: - - "null" - - array - date: - $ref: "#/$defs/date.json" - number: - type: - - "null" - - number - type: - enum: - - number - - date - - array - type: - - "null" - - object - type: - enum: - - rollup - type: object - - properties: - id: - type: string - people: - items: - $ref: "#/$defs/user.json" - type: - - "null" - - array - type: - enum: - - people - type: object - - properties: - files: - items: - properties: - expiry_time: - type: - - "null" - - string - name: - type: string - type: - enum: - - external - - file - url: - type: string - type: object - type: - - "null" - - array - id: - type: string - type: - enum: - - files - type: object - - properties: - checkout: - type: - - "null" - - boolean - id: - type: string - type: - enum: - - checkbox - type: object - - properties: - id: - type: string - type: - enum: - - url - url: - type: string - type: object - - properties: - email: - type: string - id: - type: string - type: - enum: - - email - type: object - - properties: - id: - type: string - phone_number: - type: string - type: - enum: - - phone_number - type: object - - properties: - created_time: - type: string - id: - type: string - type: - enum: - - created_time - type: object - - properties: - created_by: - $ref: "#/$defs/user.json" - id: - type: string - type: - enum: - - created_by - type: object - - properties: - id: - type: string - last_edited_time: - type: string - type: - enum: - - last_edited_time - type: object - - properties: - id: - type: string - last_edited_by: - $ref: "#/$defs/user.json" - type: - enum: - - last_edited_by - type: object - type: object - type: object - type: array url: type: string -required: - - id -type: object + public_url: + type: + - "null" + - string + properties: + type: array + _meta: + type: object + properties: + row_id: + type: integer + required: + - row_id +x-infer-schema: true diff --git a/source-notion/acmeCo/users.schema.yaml b/source-notion/acmeCo/users.schema.yaml index 9e85d079e1..0e37656415 100644 --- a/source-notion/acmeCo/users.schema.yaml +++ b/source-notion/acmeCo/users.schema.yaml @@ -1,42 +1,93 @@ --- $schema: "http://json-schema.org/draft-07/schema#" +type: object additionalProperties: true properties: + object: + enum: + - user + id: + type: string + name: + type: + - "null" + - string avatar_url: type: - "null" - string + type: + enum: + - person + - bot + person: + type: + - "null" + - object + additionalProperties: true + properties: + email: + type: + - "null" + - string bot: + type: + - "null" + - object additionalProperties: true properties: owner: + type: object properties: type: type: string + info: + type: + - object + - boolean + properties: + avatar_url: + type: + - "null" + - string + id: + type: + - "null" + - string + name: + type: + - "null" + - string + object: + type: + - "null" + - string + person: + type: + - "null" + - object + properties: + email: + type: + - "null" + - string + type: + type: + - "null" + - string workspace: type: - "null" - boolean - type: object - type: - - "null" - - object - id: - type: string - name: - type: string - object: - type: string - person: - additionalProperties: true + workspace_name: + type: + - "null" + - string + _meta: + type: object properties: - email: - type: string - type: - - "null" - - object - type: - type: string -required: - - id -type: object + row_id: + type: integer + required: + - row_id +x-infer-schema: true diff --git a/source-notion/source_notion/__main__.py b/source-notion/source_notion/__main__.py new file mode 100644 index 0000000000..f913e51c8b --- /dev/null +++ b/source-notion/source_notion/__main__.py @@ -0,0 +1,46 @@ +import estuary_cdk.pydantic_polyfill # Must be first. + +import asyncio +import json + +from estuary_cdk import flow, shim_airbyte_cdk + +from source_notion import SourceNotion + + +def urlencode_field(field: str): + return "{{#urlencode}}{{{ " + field + " }}}{{/urlencode}}" + + +accessTokenBody = { + "grant_type": "authorization_code", + "redirect_uri": "{{{ redirect_uri }}}", + "code": "{{{ code }}}", +} + +asyncio.run( + shim_airbyte_cdk.CaptureShim( + delegate=SourceNotion(), + oauth2=flow.OAuth2Spec( + provider="notion", + accessTokenBody=json.dumps(accessTokenBody), + authUrlTemplate=( + f"https://api.notion.com/v1/oauth/authorize?" + f"client_id={urlencode_field('client_id')}&" + f"redirect_uri={urlencode_field('redirect_uri')}&" + f"response_type=code&" + f"owner=user&" + f"state={urlencode_field('state')}" + ), + accessTokenUrlTemplate=(f"https://api.notion.com/v1/oauth/token"), + accessTokenResponseMap={ + "access_token": "/access_token", + }, + accessTokenHeaders={ + "Authorization": "Basic {{#basicauth}}{{{ client_id }}}:{{{ client_secret }}}{{/basicauth}}", + "Content-Type": "application/json", + }, + ), + schema_inference=True, + ).serve() +) diff --git a/source-notion/source_notion/schemas/databases.json b/source-notion/source_notion/schemas/databases.json index 55f004c522..070fc3c0e6 100644 --- a/source-notion/source_notion/schemas/databases.json +++ b/source-notion/source_notion/schemas/databases.json @@ -57,499 +57,7 @@ "type": ["null", "string"] }, "properties": { - "type": "array", - "items": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": "object", - "additionalProperties": true, - "anyOf": [ - { - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": [ - "checkbox", - "created_by", - "created_time", - "date", - "email", - "files", - "formula", - "last_edited_by", - "last_edited_time", - "multi_select", - "number", - "people", - "phone_number", - "relation", - "rich_text", - "rollup", - "select", - "status", - "title", - "url" - ] - }, - "name": { - "type": "string" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["checkbox"] - }, - "name": { - "type": "string" - }, - "checkbox": { - "type": "object" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["created_by"] - }, - "name": { - "type": "string" - }, - "created_by": { - "type": "object" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["created_time"] - }, - "name": { - "type": "string" - }, - "created_time": { - "type": "object" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["date"] - }, - "name": { - "type": "string" - }, - "date": { - "type": "object" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["email"] - }, - "name": { - "type": "string" - }, - "email": { - "type": "object" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["files"] - }, - "name": { - "type": "string" - }, - "files": { - "type": "object" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["formula"] - }, - "name": { - "type": "string" - }, - "expression": { - "type": "string" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["last_edited_by"] - }, - "name": { - "type": "string" - }, - "last_edited_by": { - "type": "string" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["last_edited_time"] - }, - "name": { - "type": "string" - }, - "last_edited_time": { - "type": "string" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["number"] - }, - "name": { - "type": "string" - }, - "format": { - "enum": [ - "argentine_peso", - "baht", - "canadian_dollar", - "chilean_peso", - "colombian_peso", - "danish_krone", - "dirham", - "dollar", - "euro", - "forint", - "franc", - "hong_kong_dollar", - "koruna", - "krona", - "leu", - "lira", - "mexican_peso", - "new_taiwan_dollar", - "new_zealand_dollar", - "norwegian_krone", - "number", - "number_with_commas", - "percent", - "peruvian_sol", - "philippine_peso", - "pound", - "rand", - "real", - "ringgit", - "riyal", - "ruble", - "rupee", - "rupiah", - "shekel", - "singapore_dollar", - "uruguayan_peso", - "won", - "yen", - "yuan", - "zloty" - ] - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["select", "multi_select"] - }, - "name": { - "type": "string" - }, - "options": { - "type": "array", - "items": { - "$ref": "options.json" - } - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["people"] - }, - "name": { - "type": "string" - }, - "people": { - "type": "string" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["phone_number"] - }, - "name": { - "type": "string" - }, - "phone_number": { - "type": "object" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["relation"] - }, - "name": { - "type": "string" - }, - "database_id": { - "type": "string" - }, - "synced_property_name": { - "type": ["null", "string"] - }, - "synced_property_id": { - "type": ["null", "string"] - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["rich_text"] - }, - "name": { - "type": "string" - }, - "rich_text": { - "type": "object" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["rollup"] - }, - "name": { - "type": "string" - }, - "relation_property_name": { - "type": "string" - }, - "relation_property_id": { - "type": "string" - }, - "rollup_property_name": { - "type": "string" - }, - "rollup_property_id": { - "type": "string" - }, - "function": { - "enum": [ - "average", - "checked", - "count_per_group", - "count", - "count_values", - "date_range", - "earliest_date", - "empty", - "latest_date", - "max", - "median", - "min", - "not_empty", - "percent_checked", - "percent_empty", - "percent_not_empty", - "percent_per_group", - "percent_unchecked", - "range", - "unchecked", - "unique", - "show_original", - "show_unique", - "sum" - ] - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["status"] - }, - "name": { - "type": "string" - }, - "status": { - "type": "object", - "properties": { - "options": { - "type": "array", - "items": { - "$ref": "options.json" - } - }, - "groups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "color": { - "type": "string" - }, - "option_ids": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - } - } - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["title"] - }, - "name": { - "type": "string" - }, - "title": { - "type": "object" - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["url"] - }, - "name": { - "type": "string" - }, - "url": { - "type": "object" - } - } - } - ] - } - } - } + "type": "array" } } } diff --git a/source-notion/source_notion/schemas/pages.json b/source-notion/source_notion/schemas/pages.json index 7972b07d6c..48fdeec576 100644 --- a/source-notion/source_notion/schemas/pages.json +++ b/source-notion/source_notion/schemas/pages.json @@ -42,289 +42,7 @@ "type": ["null", "string"] }, "properties": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": true, - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": "object", - "additionalProperties": true, - "oneOf": [ - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["title"] - }, - "title": { "$ref": "title.json" } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["rich_text"] - }, - "rich_text": { - "type": ["null", "array"], - "items": { "$ref": "rich_text.json" } - } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["select"] }, - "select": { "$ref": "options.json" } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["multi_select"] }, - "multi_select": { - "type": ["null", "array"], - "items": { "$ref": "options.json" } - } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["date"] }, - "date": { "$ref": "date.json" } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["formula"] }, - "formula": { - "type": ["null", "object"], - "properties": { - "type": { - "enum": ["string", "number", "boolean", "date"] - }, - "string": { "type": ["null", "string"] }, - "number": { "type": ["null", "number"] }, - "boolean": { "type": ["null", "boolean"] }, - "date": { "$ref": "date.json" } - } - } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["relation"] }, - "relation": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "id": { "type": "string" } - } - } - } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["rollup"] }, - "rollup": { - "type": ["null", "object"], - "properties": { - "type": { "enum": ["number", "date", "array"] }, - "number": { "type": ["null", "number"] }, - "date": { "$ref": "date.json" }, - "array": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "type": { "type": "string" } - } - } - } - } - } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["people"] }, - "people": { - "type": ["null", "array"], - "items": { - "$ref": "user.json" - } - } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["files"] }, - "files": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "type": { "enum": ["external", "file"] }, - "url": { "type": "string" }, - "expiry_time": { "type": ["null", "string"] }, - "name": { "type": "string" } - } - } - } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["checkbox"] }, - "checkout": { - "type": ["null", "boolean"] - } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["url"] }, - "url": { "type": "string" } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["email"] }, - "email": { "type": "string" } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["phone_number"] }, - "phone_number": { "type": "object" } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["created_time"] }, - "created_time": { "type": "string" } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["created_by"] }, - "created_by": { "$ref": "user.json" } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["last_edited_time"] }, - "last_edited_time": { "type": "string" } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["last_edited_by"] }, - "last_edited_by": { "$ref": "user.json" } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["number"] }, - "number": { - "type": "object", - "properties": { - "format": { "type": "string" } - } - } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["status"] }, - "status": { "$ref": "options.json" } - } - }, - { - "type": "object", - "properties": { - "id": { "type": "string" }, - "type": { "enum": ["unique_id"] }, - "unique_id": { - "type": "object", - "properties": { - "number": { "type": "number" }, - "prefix": { "type": ["null", "string"] } - } - } - } - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["verification"] - }, - "verification": { - "type": "object", - "properties": { - "state": { - "enum": ["verified", "unverified"] - }, - "verified_by": { "$ref": "user.json" }, - "date": { "$ref": "date.json" } - } - } - } - } - ] - } - } - } + "type": "array" } } } diff --git a/source-notion/source_notion/schemas/shared/user.json b/source-notion/source_notion/schemas/shared/user.json index 8c5000dba2..f302eb338b 100644 --- a/source-notion/source_notion/schemas/shared/user.json +++ b/source-notion/source_notion/schemas/shared/user.json @@ -38,26 +38,29 @@ "type": "string" }, "info": { - "avatar_url": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "person": { - "type": ["null", "object"], - "properties": { - "email": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] + "type": ["object", "boolean"], + "properties": { + "avatar_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "person": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } } } } diff --git a/source-notion/source_notion/spec.json b/source-notion/source_notion/spec.json index 4b833a5674..ebb4429866 100644 --- a/source-notion/source_notion/spec.json +++ b/source-notion/source_notion/spec.json @@ -1,5 +1,5 @@ { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/notion", + "documentationUrl": "https://go.estuary.dev/source-notion", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Notion Source Spec", @@ -17,13 +17,17 @@ }, "credentials": { "title": "Authentication Method", - "description": "Choose either OAuth (recommended for Airbyte Cloud) or Access Token. See our docs for more information.", + "description": "Choose either OAuth or Access Token.", + "discriminator": { + "propertyName": "auth_type" + }, "type": "object", "order": 1, "oneOf": [ { "type": "object", "title": "OAuth2.0", + "x-oauth2-provider": "notion", "required": [ "auth_type", "client_id", @@ -38,19 +42,19 @@ "client_id": { "title": "Client ID", "type": "string", - "description": "The Client ID of your Notion integration. See our docs for more information.", + "description": "The Client ID of your Notion integration.", "airbyte_secret": true }, "client_secret": { "title": "Client Secret", "type": "string", - "description": "The Client Secret of your Notion integration. See our docs for more information.", + "description": "The Client Secret of your Notion integration.", "airbyte_secret": true }, "access_token": { "title": "Access Token", "type": "string", - "description": "The Access Token received by completing the OAuth flow for your Notion integration. See our docs for more information.", + "description": "The Access Token received by completing the OAuth flow for your Notion integration.", "airbyte_secret": true } } @@ -66,7 +70,7 @@ }, "token": { "title": "Access Token", - "description": "The Access Token for your private Notion integration. See the docs for more information on how to obtain this token.", + "description": "The Access Token for your private Notion integration. See the docs for more information: https://go.estuary.dev/u5BKFR", "type": "string", "airbyte_secret": true } diff --git a/source-notion/test.flow.yaml b/source-notion/test.flow.yaml index 72f1c1964c..2b5b58ac3c 100644 --- a/source-notion/test.flow.yaml +++ b/source-notion/test.flow.yaml @@ -4,15 +4,16 @@ import: captures: acmeCo/source-notion: endpoint: - connector: - image: "ghcr.io/estuary/source-notion:v2" + local: + command: + - python + - "-m" + - source_notion config: config.yaml bindings: - resource: stream: users syncMode: full_refresh - cursorField: - - id target: acmeCo/users - resource: stream: databases @@ -32,3 +33,9 @@ captures: cursorField: - last_edited_time target: acmeCo/blocks + - resource: + stream: comments + syncMode: incremental + cursorField: + - page_last_edited_time + target: acmeCo/comments diff --git a/source-notion/tests/snapshots/snapshots__capture__capture.stdout.json b/source-notion/tests/snapshots/snapshots__capture__capture.stdout.json index b0ed06fb0a..0f05b2b2cb 100644 --- a/source-notion/tests/snapshots/snapshots__capture__capture.stdout.json +++ b/source-notion/tests/snapshots/snapshots__capture__capture.stdout.json @@ -2,6 +2,10 @@ [ "acmeCo/users", { + "_meta": { + "op": "u", + "row_id": 0 + }, "avatar_url": null, "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", "name": "WillB Estuary", @@ -13,11 +17,15 @@ [ "acmeCo/users", { + "_meta": { + "op": "u", + "row_id": 1 + }, "avatar_url": null, "bot": { "owner": { - "type": "workspace", - "workspace": true + "info": true, + "type": "workspace" }, "workspace_name": "WillB Estuary's Notion" }, @@ -30,6 +38,10 @@ [ "acmeCo/databases", { + "_meta": { + "op": "u", + "row_id": 0 + }, "archived": false, "cover": null, "created_by": { @@ -117,6 +129,10 @@ [ "acmeCo/pages", { + "_meta": { + "op": "u", + "row_id": 1 + }, "archived": false, "cover": null, "created_by": { @@ -169,6 +185,10 @@ [ "acmeCo/pages", { + "_meta": { + "op": "u", + "row_id": 2 + }, "archived": false, "cover": null, "created_by": { @@ -223,6 +243,10 @@ [ "acmeCo/pages", { + "_meta": { + "op": "u", + "row_id": 4 + }, "archived": false, "cover": null, "created_by": { @@ -293,6 +317,10 @@ [ "acmeCo/pages", { + "_meta": { + "op": "u", + "row_id": 0 + }, "archived": false, "cover": null, "created_by": { @@ -347,6 +375,10 @@ [ "acmeCo/pages", { + "_meta": { + "op": "u", + "row_id": 3 + }, "archived": false, "cover": null, "created_by": { @@ -423,6 +455,10 @@ [ "acmeCo/blocks", { + "_meta": { + "op": "u", + "row_id": 1 + }, "archived": false, "created_by": { "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", @@ -451,6 +487,10 @@ [ "acmeCo/blocks", { + "_meta": { + "op": "u", + "row_id": 2 + }, "archived": false, "created_by": { "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", @@ -497,6 +537,10 @@ [ "acmeCo/blocks", { + "_meta": { + "op": "u", + "row_id": 5 + }, "archived": false, "created_by": { "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", @@ -544,6 +588,10 @@ [ "acmeCo/blocks", { + "_meta": { + "op": "u", + "row_id": 0 + }, "archived": false, "created_by": { "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", @@ -590,6 +638,10 @@ [ "acmeCo/blocks", { + "_meta": { + "op": "u", + "row_id": 4 + }, "archived": false, "created_by": { "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", @@ -637,6 +689,10 @@ [ "acmeCo/blocks", { + "_meta": { + "op": "u", + "row_id": 6 + }, "archived": false, "created_by": { "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", @@ -665,6 +721,10 @@ [ "acmeCo/blocks", { + "_meta": { + "op": "u", + "row_id": 7 + }, "archived": false, "created_by": { "id": "2ab751df-b6a3-40ba-a547-44c5306c0248", @@ -693,6 +753,10 @@ [ "acmeCo/blocks", { + "_meta": { + "op": "u", + "row_id": 3 + }, "archived": false, "code": { "caption": [], diff --git a/source-notion/tests/snapshots/snapshots__discover__capture.stdout.json b/source-notion/tests/snapshots/snapshots__discover__capture.stdout.json index df88879037..ebbb072b4b 100644 --- a/source-notion/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-notion/tests/snapshots/snapshots__discover__capture.stdout.json @@ -3,73 +3,148 @@ "recommendedName": "users", "resourceConfig": { "stream": "users", - "syncMode": "full_refresh", - "cursorField": [ - "id" - ] + "syncMode": "full_refresh" }, "documentSchema": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "additionalProperties": true, "properties": { + "object": { + "enum": [ + "user" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": [ + "null", + "string" + ] + }, "avatar_url": { "type": [ "null", "string" ] }, + "type": { + "enum": [ + "person", + "bot" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "email": { + "type": [ + "null", + "string" + ] + } + } + }, "bot": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "owner": { + "type": "object", "properties": { "type": { "type": "string" }, + "info": { + "type": [ + "object", + "boolean" + ], + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "properties": { + "email": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, "workspace": { "type": [ "null", "boolean" ] } - }, - "type": "object" + } + }, + "workspace_name": { + "type": [ + "null", + "string" + ] } - }, - "type": [ - "null", - "object" - ] - }, - "id": { - "type": "string" - }, - "name": { - "type": "string" + } }, - "object": { - "type": "string" - }, - "person": { - "additionalProperties": true, + "_meta": { + "type": "object", "properties": { - "email": { - "type": "string" + "row_id": { + "type": "integer" } }, - "type": [ - "null", - "object" + "required": [ + "row_id" ] - }, - "type": { - "type": "string" } }, - "required": [ - "id" - ], - "type": "object" + "x-infer-schema": true }, "key": [ "/id" @@ -85,357 +160,638 @@ ] }, "documentSchema": { - "$defs": { - "options.json": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "color": { - "enum": [ - "default", - "gray", - "brown", - "orange", - "yellow", - "green", - "blue", - "purple", - "pink", - "red" - ] - }, - "id": { - "type": "string" - }, - "name": { - "type": "string" - } - }, - "type": [ - "null", - "object" - ] - } - }, "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "additionalProperties": true, "properties": { - "archived": { - "type": "boolean" + "object": { + "enum": [ + "database" + ] }, - "cover": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "caption": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", + "id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "last_edited_time": { + "type": "string", + "format": "date-time" + }, + "title": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "text": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, "properties": { - "annotations": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { - "bold": { - "type": [ - "null", - "boolean" - ] - }, - "code": { - "type": [ - "null", - "boolean" + "type": { + "enum": [ + "url" ] }, - "color": { + "url": { "type": [ "null", "string" ] - }, - "italic": { - "type": [ - "null", - "boolean" - ] - }, - "strikethrough": { - "type": [ - "null", - "boolean" - ] - }, - "underline": { - "type": [ - "null", - "boolean" - ] } - }, - "type": [ - "null", - "object" - ] - }, - "href": { + } + } + } + }, + "mention": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { "type": [ "null", "string" ] }, - "plain_text": { + "info": { "type": [ "null", - "string" - ] - }, - "rich_text": { - "additionalProperties": true, + "object" + ], "properties": { - "content": { + "id": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, - "properties": { - "type": { - "type": "string" - }, - "url": { - "type": [ - "null", - "string" - ] - } - }, + "object": { "type": [ "null", - "object" + "string" ] } - }, + } + } + } + }, + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "expression": { "type": [ "null", - "object" + "string" ] - }, - "text": { - "additionalProperties": true, - "properties": { - "content": { - "type": [ - "null", - "string" + } + } + }, + "annotations": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + } + } + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "description": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "text": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" ] }, - "link": { - "additionalProperties": true, - "properties": { - "type": { - "type": "string" - }, - "url": { - "type": [ - "null", - "string" - ] - } - }, + "url": { "type": [ "null", - "object" + "string" ] } - }, + } + } + } + }, + "mention": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { "type": [ "null", - "object" + "string" ] }, - "type": { + "info": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "expression": { "type": [ "null", "string" ] } - }, - "type": "object" - }, - "type": [ - "null", - "array" - ] - }, - "external": { - "additionalProperties": true, - "properties": { - "url": { - "type": "string" } }, - "type": [ - "null", - "object" - ] - }, - "file": { - "additionalProperties": true, - "properties": { - "expiry_time": { - "type": "string" - }, - "url": { - "type": "string" + "annotations": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + } } }, - "type": [ - "null", - "object" - ] - }, - "type": { - "type": "string" + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + } } - }, - "type": [ - "null", - "object" - ] + } }, - "created_by": { + "last_edited_by": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "additionalProperties": true, "properties": { - "avatar_url": { + "object": { + "enum": [ + "user" + ] + }, + "id": { + "type": "string" + }, + "name": { "type": [ "null", "string" ] }, - "bot": { - "additionalProperties": true, - "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "type": { + "enum": [ + "person", + "bot" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "email": { + "type": [ + "null", + "string" + ] + } + } + }, + "bot": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { "owner": { + "type": "object", "properties": { "type": { "type": "string" }, + "info": { + "type": [ + "object", + "boolean" + ], + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "properties": { + "email": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, "workspace": { "type": [ "null", "boolean" ] } - }, - "type": "object" + } + }, + "workspace_name": { + "type": [ + "null", + "string" + ] } - }, - "type": [ - "null", - "object" + } + } + } + }, + "created_by": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": [ + "user" ] }, "id": { "type": "string" }, "name": { - "type": "string" + "type": [ + "null", + "string" + ] }, - "object": { - "type": "string" + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "type": { + "enum": [ + "person", + "bot" + ] }, "person": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "email": { - "type": "string" + "type": [ + "null", + "string" + ] } - }, + } + }, + "bot": { "type": [ "null", "object" - ] - }, - "type": { - "type": "string" + ], + "additionalProperties": true, + "properties": { + "owner": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "info": { + "type": [ + "object", + "boolean" + ], + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "properties": { + "email": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "workspace": { + "type": [ + "null", + "boolean" + ] + } + } + }, + "workspace_name": { + "type": [ + "null", + "string" + ] + } + } } - }, - "type": "object" + } }, - "created_time": { - "type": "string" + "archived": { + "type": "boolean" }, "icon": { "$schema": "http://json-schema.org/draft-07/schema#", "anyOf": [ { "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { + "type": { + "enum": [ + "file", + "external" + ] + }, "caption": { + "type": [ + "null", + "array" + ], "items": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "properties": { - "annotations": { - "additionalProperties": true, - "properties": { - "bold": { - "type": [ - "null", - "boolean" - ] - }, - "code": { - "type": [ - "null", - "boolean" - ] - }, - "color": { - "type": [ - "null", - "string" - ] - }, - "italic": { - "type": [ - "null", - "boolean" - ] - }, - "strikethrough": { - "type": [ - "null", - "boolean" - ] - }, - "underline": { - "type": [ - "null", - "boolean" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "href": { + "type": { "type": [ "null", "string" ] }, - "plain_text": { + "text": { "type": [ "null", - "string" - ] - }, - "rich_text": { + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -445,6 +801,10 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { @@ -458,1066 +818,985 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "enum": [ - "url" + "id": { + "type": [ + "null", + "string" ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ + } + } + } + }, + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ "null", - "object" + "string" ] } - }, + } + }, + "annotations": { "type": [ "null", "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + } + } + }, + "plain_text": { + "type": [ + "null", + "string" ] }, - "type": { + "href": { "type": [ "null", "string" ] } - }, - "type": "object" - }, - "type": [ - "null", - "array" - ] + } + } }, "external": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "url": { "type": "string" } - }, + } + }, + "file": { "type": [ "null", "object" - ] - }, - "file": { + ], "additionalProperties": true, "properties": { - "expiry_time": { - "type": "string" - }, "url": { "type": "string" + }, + "expiry_time": { + "type": "string", + "format": "date-time" } - }, - "type": [ - "null", - "object" - ] - }, - "type": { - "enum": [ - "file", - "external" - ] + } } - }, - "type": [ - "null", - "object" - ] + } }, { "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { - "emoji": { + "type": { "type": "string" }, - "type": { + "emoji": { "type": "string" } - }, - "type": [ - "null", - "object" - ] + } } ] }, - "id": { - "type": "string" - }, - "is_inline": { + "cover": { + "$schema": "http://json-schema.org/draft-07/schema#", "type": [ "null", - "boolean" - ] - }, - "last_edited_by": { - "$schema": "http://json-schema.org/draft-07/schema#", + "object" + ], "additionalProperties": true, "properties": { - "avatar_url": { - "type": [ - "null", - "string" - ] - }, - "bot": { - "additionalProperties": true, - "properties": { - "owner": { - "properties": { - "type": { - "type": "string" - }, - "workspace": { - "type": [ - "null", - "boolean" - ] - } - }, - "type": "object" - } - }, - "type": [ - "null", - "object" + "type": { + "enum": [ + "file", + "external" ] }, - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "object": { - "type": "string" - }, - "person": { - "additionalProperties": true, - "properties": { - "email": { - "type": "string" - } - }, + "caption": { "type": [ "null", - "object" - ] - }, - "type": { - "type": "string" - } - }, - "type": "object" - }, - "last_edited_time": { - "type": "string" - }, - "object": { - "type": "string" - }, - "parent": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "database_id": { - "type": "string" - }, - "page_id": { - "type": "string" - }, - "type": { - "type": "string" - }, - "workspace": { - "type": "boolean" - } - }, - "type": "object" - }, - "properties": { - "items": { - "additionalProperties": true, - "properties": { - "name": { - "type": [ - "null", - "string" - ] - }, - "value": { - "additionalProperties": true, - "anyOf": [ - { - "properties": { - "id": { - "type": "string" - }, - "type": { - "type": "string" - } - }, - "type": "object" + "array" + ], + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": [ + "null", + "string" + ] }, - { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "type": { - "enum": [ - "title", - "rich_text", - "date", - "people", - "files", - "checkbox", - "url", - "email", - "phone_number", - "created_time", - "created_by", - "last_edited_time", - "last_edited_by" - ] - } - }, - "type": "object" - }, - { - "properties": { - "format": { - "enum": [ - "number", - "number_with_commas", - "percent", - "dollar", - "canadian_dollar", - "euro", - "pound", - "yen", - "ruble", - "rupee", - "won", - "yuan", - "real", - "lira", - "rupiah", - "franc", - "hong_kong_dollar", - "new_zealand_dollar", - "krona", - "norwegian_krone", - "mexican_peso", - "rand", - "new_taiwan_dollar", - "danish_krone", - "zloty", - "baht", - "forint", - "koruna", - "shekel", - "chilean_peso", - "philippine_peso", - "dirham", - "colombian_peso", - "riyal", - "ringgit", - "leu" + "content": { + "type": [ + "null", + "string" ] }, - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "type": { - "enum": [ - "number" - ] + "link": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } } - }, - "type": "object" + } }, - { + "mention": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "options": { - "items": { - "$ref": "#/$defs/options.json" - }, - "type": "array" - }, "type": { - "enum": [ - "select", - "multi_select" + "type": [ + "null", + "string" ] + }, + "info": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + } + } } - }, - "type": "object" + } }, - { + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, "properties": { "expression": { - "type": "string" - }, - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "type": { - "enum": [ - "formula" + "type": [ + "null", + "string" ] } - }, - "type": "object" + } }, - { + "annotations": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, "properties": { - "database_id": { - "type": "string" - }, - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "synced_property_id": { + "bold": { "type": [ "null", - "string" + "boolean" ] }, - "synced_property_name": { + "italic": { "type": [ "null", - "string" + "boolean" ] }, - "type": { - "enum": [ - "relation" - ] - } - }, - "type": "object" - }, - { - "properties": { - "function": { - "enum": [ - "count_all", - "count_values", - "count_unique_values", - "count_empty", - "count_not_empty", - "percent_empty", - "percent_not_empty", - "sum", - "average", - "median", - "min", - "max", - "range", - "show_original" + "strikethrough": { + "type": [ + "null", + "boolean" ] }, - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "relation_property_id": { - "type": "string" - }, - "relation_property_name": { - "type": "string" - }, - "rollup_property_id": { - "type": "string" - }, - "rollup_property_name": { - "type": "string" + "underline": { + "type": [ + "null", + "boolean" + ] }, - "type": { - "enum": [ - "rollup" + "code": { + "type": [ + "null", + "boolean" ] - } - }, - "type": "object" - } - ], - "type": "object" - } - }, - "type": [ - "null", - "object" - ] - }, - "type": "array" - }, - "title": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { - "additionalProperties": true, - "properties": { - "bold": { - "type": [ - "null", - "boolean" - ] - }, - "code": { - "type": [ - "null", - "boolean" - ] - }, - "color": { - "type": [ - "null", - "string" - ] - }, - "italic": { - "type": [ - "null", - "boolean" - ] - }, - "strikethrough": { - "type": [ - "null", - "boolean" - ] - }, - "underline": { - "type": [ - "null", - "boolean" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "href": { - "type": [ - "null", - "string" - ] - }, - "plain_text": { - "type": [ - "null", - "string" - ] - }, - "rich_text": { - "additionalProperties": true, - "properties": { - "content": { - "type": [ - "null", - "string" - ] - }, - "link": { - "additionalProperties": true, - "properties": { - "type": { - "type": "string" }, - "url": { + "color": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "text": { - "additionalProperties": true, - "properties": { - "content": { + } + }, + "plain_text": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, - "properties": { - "type": { - "type": "string" - }, - "url": { - "type": [ - "null", - "string" - ] - } - }, + "href": { "type": [ "null", - "object" + "string" ] } - }, - "type": [ - "null", - "object" - ] - }, - "type": { - "type": [ - "null", - "string" - ] + } } }, - "type": "object" - }, - "type": "array" - }, - "url": { - "type": "string" - } - }, - "required": [ - "id" - ], - "type": "object" - }, - "key": [ - "/id" - ] - }, - { - "recommendedName": "pages", - "resourceConfig": { - "stream": "pages", - "syncMode": "incremental", - "cursorField": [ - "last_edited_time" - ] - }, - "documentSchema": { - "$defs": { - "date.json": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "end": { - "type": [ - "null", - "string" - ] - }, - "start": { + "external": { "type": [ "null", - "string" - ] + "object" + ], + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + } }, - "time_zone": { + "file": { "type": [ "null", - "string" - ] + "object" + ], + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + }, + "expiry_time": { + "type": "string", + "format": "date-time" + } + } } - }, - "type": [ - "null", - "object" - ] + } }, - "options.json": { + "parent": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "additionalProperties": true, "properties": { - "color": { + "type": { "enum": [ - "default", - "gray", - "brown", - "orange", - "yellow", - "green", - "blue", - "purple", - "pink", - "red" + "block_id", + "database_id", + "page_id", + "workspace" ] }, - "id": { + "block_id": { "type": "string" }, - "name": { + "database_id": { + "type": "string" + }, + "page_id": { "type": "string" + }, + "workspace": { + "type": "boolean" } - }, + } + }, + "url": { + "type": "string" + }, + "is_inline": { "type": [ "null", - "object" + "boolean" + ] + }, + "public_url": { + "type": [ + "null", + "string" + ] + }, + "properties": { + "type": "array" + }, + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "required": [ + "row_id" + ] + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, + { + "recommendedName": "pages", + "resourceConfig": { + "stream": "pages", + "syncMode": "incremental", + "cursorField": [ + "last_edited_time" + ] + }, + "documentSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": [ + "page" ] }, - "rich_text.json": { + "id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "created_by": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, "properties": { - "annotations": { - "additionalProperties": true, - "properties": { - "bold": { - "type": [ - "null", - "boolean" - ] - }, - "code": { - "type": [ - "null", - "boolean" - ] - }, - "color": { - "type": [ - "null", - "string" - ] - }, - "italic": { - "type": [ - "null", - "boolean" - ] - }, - "strikethrough": { - "type": [ - "null", - "boolean" - ] - }, - "underline": { - "type": [ - "null", - "boolean" - ] - } - }, - "type": [ - "null", - "object" + "object": { + "enum": [ + "user" ] }, - "href": { + "id": { + "type": "string" + }, + "name": { "type": [ "null", "string" ] }, - "plain_text": { + "avatar_url": { "type": [ "null", "string" ] }, - "rich_text": { + "type": { + "enum": [ + "person", + "bot" + ] + }, + "person": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { - "content": { + "email": { "type": [ "null", "string" ] - }, - "link": { - "additionalProperties": true, - "properties": { - "type": { - "enum": [ - "url" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] } - }, + } + }, + "bot": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { - "type": [ - "null", - "string" - ] - }, - "link": { - "additionalProperties": true, + "owner": { + "type": "object", "properties": { "type": { - "enum": [ - "url" - ] + "type": "string" }, - "url": { + "info": { + "type": [ + "object", + "boolean" + ], + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "properties": { + "email": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "workspace": { "type": [ "null", - "string" + "boolean" ] } - }, + } + }, + "workspace_name": { "type": [ "null", - "object" + "string" ] } - }, + } + } + } + }, + "last_edited_time": { + "type": "string", + "format": "date-time" + }, + "last_edited_by": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": [ + "user" + ] + }, + "id": { + "type": "string" + }, + "name": { "type": [ "null", - "object" + "string" ] }, - "type": { + "avatar_url": { "type": [ "null", "string" ] - } - }, - "type": "object" - }, - "title.json": { - "items": { - "additionalProperties": true, - "properties": { - "annotations": { - "additionalProperties": true, - "properties": { - "bold": { - "type": [ - "null", - "boolean" - ] - }, - "code": { - "type": [ - "null", - "boolean" - ] - }, - "color": { - "type": [ - "null", - "string" - ] - }, - "italic": { - "type": [ - "null", - "boolean" - ] - }, - "strikethrough": { - "type": [ - "null", - "boolean" - ] - }, - "underline": { - "type": [ - "null", - "boolean" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "href": { - "type": [ - "null", - "string" - ] - }, - "plain_text": { - "type": [ - "null", - "string" - ] - }, - "text": { - "properties": { - "content": { - "type": [ - "null", - "string" - ] - }, - "link": { - "additionalProperties": true, - "properties": { - "type": { - "enum": [ - "url" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - } }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "user.json": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "avatar_url": { + "type": { + "enum": [ + "person", + "bot" + ] + }, + "person": { "type": [ "null", - "string" - ] + "object" + ], + "additionalProperties": true, + "properties": { + "email": { + "type": [ + "null", + "string" + ] + } + } }, "bot": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "owner": { + "type": "object", "properties": { "type": { "type": "string" }, + "info": { + "type": [ + "object", + "boolean" + ], + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "properties": { + "email": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, "workspace": { "type": [ "null", "boolean" ] } - }, - "type": "object" - } - }, - "type": [ - "null", - "object" - ] - }, - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "object": { - "enum": [ - "user" - ] - }, - "person": { - "additionalProperties": true, - "properties": { - "email": { - "type": "string" + } + }, + "workspace_name": { + "type": [ + "null", + "string" + ] } - }, - "type": [ - "null", - "object" - ] - }, - "type": { - "enum": [ - "person", - "bot" - ] + } } - }, - "type": "object" - } - }, - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { + } + }, "archived": { "type": "boolean" }, - "cover": { + "icon": { "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "caption": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { - "additionalProperties": true, + "anyOf": [ + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "file", + "external" + ] + }, + "caption": { + "type": [ + "null", + "array" + ], + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "properties": { - "bold": { - "type": [ - "null", - "boolean" - ] - }, - "code": { - "type": [ - "null", - "boolean" - ] - }, - "color": { + "type": { "type": [ "null", "string" ] }, - "italic": { - "type": [ - "null", - "boolean" - ] - }, - "strikethrough": { + "text": { "type": [ "null", - "boolean" - ] - }, - "underline": { - "type": [ + "object" + ], + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "mention": { + "type": [ "null", - "boolean" + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "info": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } + }, + "annotations": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + } + } + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "href": { + "type": [ + "null", + "string" ] } + } + } + }, + "external": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + } + }, + "file": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "url": { + "type": "string" }, - "type": [ - "null", - "object" - ] - }, - "href": { + "expiry_time": { + "type": "string", + "format": "date-time" + } + } + } + } + }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "type": "string" + }, + "emoji": { + "type": "string" + } + } + } + ] + }, + "cover": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "file", + "external" + ] + }, + "caption": { + "type": [ + "null", + "array" + ], + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "plain_text": { + "text": { "type": [ "null", - "string" - ] - }, - "rich_text": { + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -1527,10 +1806,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -1538,609 +1823,675 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": [ - "null", - "array" - ] + "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + } + } + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + } + } + } }, "external": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "url": { "type": "string" } - }, + } + }, + "file": { "type": [ "null", "object" - ] - }, - "file": { + ], "additionalProperties": true, "properties": { - "expiry_time": { - "type": "string" - }, "url": { "type": "string" + }, + "expiry_time": { + "type": "string", + "format": "date-time" } - }, - "type": [ - "null", - "object" + } + } + } + }, + "parent": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "block_id", + "database_id", + "page_id", + "workspace" ] }, - "type": { + "block_id": { + "type": "string" + }, + "database_id": { + "type": "string" + }, + "page_id": { "type": "string" + }, + "workspace": { + "type": "boolean" } - }, - "type": [ - "null", - "object" - ] + } }, - "created_time": { + "url": { "type": "string" }, - "id": { - "type": "string" + "public_url": { + "type": [ + "null", + "string" + ] }, - "last_edited_time": { - "type": "string" + "properties": { + "type": "array" }, + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "required": [ + "row_id" + ] + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, + { + "recommendedName": "blocks", + "resourceConfig": { + "stream": "blocks", + "syncMode": "incremental", + "cursorField": [ + "last_edited_time" + ] + }, + "documentSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { "object": { - "type": "string" + "enum": [ + "block" + ] }, "parent": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "additionalProperties": true, "properties": { - "database_id": { + "type": { + "enum": [ + "block_id", + "database_id", + "page_id", + "workspace" + ] + }, + "block_id": { "type": "string" }, - "page_id": { + "database_id": { "type": "string" }, - "type": { + "page_id": { "type": "string" }, "workspace": { "type": "boolean" } - }, - "type": "object" + } }, - "properties": { - "items": { - "additionalProperties": true, - "properties": { - "name": { - "type": [ - "null", - "string" - ] - }, - "value": { - "additionalProperties": true, - "anyOf": [ - { - "properties": { - "id": { - "type": "string" - }, - "type": { - "type": "string" - } + "id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": [ + "user" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "type": { + "enum": [ + "person", + "bot" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "email": { + "type": [ + "null", + "string" + ] + } + } + }, + "bot": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "owner": { + "type": "object", + "properties": { + "type": { + "type": "string" }, - "type": "object" - }, - { - "properties": { - "id": { - "type": "string" - }, - "title": { - "$ref": "#/$defs/title.json" - }, - "type": { - "enum": [ - "title" - ] + "info": { + "type": [ + "object", + "boolean" + ], + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "properties": { + "email": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } } }, - "type": "object" - }, - { - "properties": { - "id": { - "type": "string" - }, - "rich_text": { - "$ref": "#/$defs/rich_text.json" - }, - "type": { - "enum": [ - "rich_text" - ] - } + "workspace": { + "type": [ + "null", + "boolean" + ] + } + } + }, + "workspace_name": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "last_edited_by": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": [ + "user" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "type": { + "enum": [ + "person", + "bot" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "email": { + "type": [ + "null", + "string" + ] + } + } + }, + "bot": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "owner": { + "type": "object", + "properties": { + "type": { + "type": "string" }, - "type": "object" - }, - { - "properties": { - "id": { - "type": "string" - }, - "select": { - "$ref": "#/$defs/options.json" - }, - "type": { - "enum": [ - "select" - ] + "info": { + "type": [ + "object", + "boolean" + ], + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "properties": { + "email": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } } }, - "type": "object" + "workspace": { + "type": [ + "null", + "boolean" + ] + } + } + }, + "workspace_name": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "last_edited_time": { + "type": "string", + "format": "date-time" + }, + "archived": { + "type": "boolean" + }, + "has_children": { + "type": [ + "null", + "boolean" + ] + }, + "type": { + "enum": [ + "bookmark", + "breadcrumb", + "bulleted_list_item", + "callout", + "child_database", + "child_page", + "code", + "column", + "column_list", + "divider", + "embed", + "equation", + "file", + "heading_1", + "heading_2", + "heading_3", + "image", + "link_preview", + "link_to_page", + "numbered_list_item", + "paragraph", + "pdf", + "quote", + "synced_block", + "table", + "table_of_contents", + "table_row", + "template", + "to_do", + "toggle", + "unsupported", + "video" + ] + }, + "bookmark": { + "type": "object", + "properties": { + "url": { + "type": "string" + }, + "caption": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": [ + "null", + "string" + ] }, - { + "text": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, "properties": { - "id": { - "type": "string" - }, - "multi_select": { - "items": { - "$ref": "#/$defs/options.json" - }, + "content": { "type": [ "null", - "array" - ] - }, - "type": { - "enum": [ - "multi_select" + "string" ] - } - }, - "type": "object" - }, - { - "properties": { - "date": { - "$ref": "#/$defs/date.json" - }, - "id": { - "type": "string" }, - "type": { - "enum": [ - "date" - ] - } - }, - "type": "object" - }, - { - "properties": { - "formula": { + "link": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, "properties": { - "boolean": { - "type": [ - "null", - "boolean" - ] - }, - "date": { - "$ref": "#/$defs/date.json" - }, - "number": { - "type": [ - "null", - "number" + "type": { + "enum": [ + "url" ] }, - "string": { + "url": { "type": [ "null", "string" ] - }, - "type": { - "enum": [ - "string", - "number", - "boolean", - "date" - ] } - }, - "type": [ - "null", - "object" - ] - }, - "id": { - "type": "string" - }, - "type": { - "enum": [ - "formula" - ] + } } - }, - "type": "object" + } }, - { + "mention": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, "properties": { - "id": { - "type": "string" - }, - "relation": { - "items": { - "properties": { - "id": { - "type": "string" - } - }, - "type": "object" - }, + "type": { "type": [ "null", - "array" - ] - }, - "type": { - "enum": [ - "relation" + "string" ] - } - }, - "type": "object" - }, - { - "properties": { - "id": { - "type": "string" }, - "rollup": { + "info": { + "type": [ + "null", + "object" + ], "properties": { - "array": { - "items": { - "properties": { - "type": { - "type": "string" - } - }, - "type": "object" - }, + "id": { "type": [ "null", - "array" + "string" ] }, - "date": { - "$ref": "#/$defs/date.json" - }, - "number": { + "object": { "type": [ "null", - "number" - ] - }, - "type": { - "enum": [ - "number", - "date", - "array" + "string" ] } - }, - "type": [ - "null", - "object" - ] - }, - "type": { - "enum": [ - "rollup" - ] - } - }, - "type": "object" - }, - { - "properties": { - "id": { - "type": "string" - }, - "people": { - "items": { - "$ref": "#/$defs/user.json" - }, - "type": [ - "null", - "array" - ] - }, - "type": { - "enum": [ - "people" - ] - } - }, - "type": "object" - }, - { - "properties": { - "files": { - "items": { - "properties": { - "expiry_time": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": "string" - }, - "type": { - "enum": [ - "external", - "file" - ] - }, - "url": { - "type": "string" - } - }, - "type": "object" - }, - "type": [ - "null", - "array" - ] - }, - "id": { - "type": "string" - }, - "type": { - "enum": [ - "files" - ] - } - }, - "type": "object" - }, - { - "properties": { - "checkout": { - "type": [ - "null", - "boolean" - ] - }, - "id": { - "type": "string" - }, - "type": { - "enum": [ - "checkbox" - ] - } - }, - "type": "object" - }, - { - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": [ - "url" - ] - }, - "url": { - "type": "string" - } - }, - "type": "object" - }, - { - "properties": { - "email": { - "type": "string" - }, - "id": { - "type": "string" - }, - "type": { - "enum": [ - "email" - ] - } - }, - "type": "object" - }, - { - "properties": { - "id": { - "type": "string" - }, - "phone_number": { - "type": "string" - }, - "type": { - "enum": [ - "phone_number" - ] - } - }, - "type": "object" - }, - { - "properties": { - "created_time": { - "type": "string" - }, - "id": { - "type": "string" - }, - "type": { - "enum": [ - "created_time" - ] - } - }, - "type": "object" - }, - { - "properties": { - "created_by": { - "$ref": "#/$defs/user.json" - }, - "id": { - "type": "string" - }, - "type": { - "enum": [ - "created_by" - ] - } - }, - "type": "object" - }, - { - "properties": { - "id": { - "type": "string" - }, - "last_edited_time": { - "type": "string" - }, - "type": { - "enum": [ - "last_edited_time" - ] + } } - }, - "type": "object" + } }, - { + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, "properties": { - "id": { - "type": "string" - }, - "last_edited_by": { - "$ref": "#/$defs/user.json" - }, - "type": { - "enum": [ - "last_edited_by" - ] - } - }, - "type": "object" - } - ], - "type": "object" - } - }, - "type": "object" - }, - "type": "array" - }, - "url": { - "type": "string" - } - }, - "required": [ - "id" - ], - "type": "object" - }, - "key": [ - "/id" - ] - }, - { - "recommendedName": "blocks", - "resourceConfig": { - "stream": "blocks", - "syncMode": "incremental", - "cursorField": [ - "last_edited_time" - ] - }, - "documentSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "archived": { - "type": "boolean" - }, - "bookmark": { - "properties": { - "caption": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } + }, "annotations": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -2149,40 +2500,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -2190,14 +2543,40 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + } + } + }, + "breadcrumb": { + "type": "object" + }, + "bulleted_list_item": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "color": { + "type": "string" + }, + "rich_text": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -2207,10 +2586,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -2218,86 +2603,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - }, - "url": { - "type": "string" - } - }, - "type": "object" - }, - "bulleted_list_item": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "children": { - "items": { - "type": "object" - }, - "type": "array" - }, - "color": { - "type": "string" - }, - "text": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -2306,40 +2670,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -2347,14 +2713,41 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "children": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "callout": { + "type": "object", + "properties": { + "color": { + "type": "string" + }, + "rich_text": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -2364,10 +2757,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -2375,145 +2774,157 @@ "string" ] } - }, + } + } + } + }, + "mention": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "info": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } + }, + "annotations": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { "type": [ "null", - "object" + "boolean" ] - } - }, - "type": [ - "null", - "object" - ] - }, - "text": { - "additionalProperties": true, - "properties": { - "content": { + }, + "code": { "type": [ "null", - "string" + "boolean" ] }, - "link": { - "additionalProperties": true, - "properties": { - "type": { - "type": "string" - }, - "url": { - "type": [ - "null", - "string" - ] - } - }, + "color": { "type": [ "null", - "object" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, - "type": { + "href": { "type": [ "null", "string" ] } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": "object" - }, - "callout": { - "properties": { - "children": { - "items": { - "type": "object" - }, - "type": "array" - }, - "color": { - "type": "string" + } + } }, "icon": { "$schema": "http://json-schema.org/draft-07/schema#", "anyOf": [ { "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { + "type": { + "enum": [ + "file", + "external" + ] + }, "caption": { + "type": [ + "null", + "array" + ], "items": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "properties": { - "annotations": { - "additionalProperties": true, - "properties": { - "bold": { - "type": [ - "null", - "boolean" - ] - }, - "code": { - "type": [ - "null", - "boolean" - ] - }, - "color": { - "type": [ - "null", - "string" - ] - }, - "italic": { - "type": [ - "null", - "boolean" - ] - }, - "strikethrough": { - "type": [ - "null", - "boolean" - ] - }, - "underline": { - "type": [ - "null", - "boolean" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "href": { + "type": { "type": [ "null", "string" ] }, - "plain_text": { + "text": { "type": [ "null", - "string" - ] - }, - "rich_text": { + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -2523,6 +2934,10 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { @@ -2536,186 +2951,210 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "enum": [ - "url" + "id": { + "type": [ + "null", + "string" ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, + } + } + } + }, + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "expression": { "type": [ "null", - "object" + "string" ] } - }, + } + }, + "annotations": { "type": [ "null", "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + } + } + }, + "plain_text": { + "type": [ + "null", + "string" ] }, - "type": { + "href": { "type": [ "null", "string" ] } - }, - "type": "object" - }, - "type": [ - "null", - "array" - ] + } + } }, "external": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "url": { "type": "string" } - }, + } + }, + "file": { "type": [ "null", "object" - ] - }, - "file": { + ], "additionalProperties": true, "properties": { - "expiry_time": { - "type": "string" - }, "url": { "type": "string" + }, + "expiry_time": { + "type": "string", + "format": "date-time" } - }, - "type": [ - "null", - "object" - ] - }, - "type": { - "enum": [ - "file", - "external" - ] - } - }, - "type": [ - "null", - "object" - ] + } + } + } }, { "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { - "emoji": { + "type": { "type": "string" }, - "type": { + "emoji": { "type": "string" } - }, - "type": [ - "null", - "object" - ] + } } ] - }, - "text": { + } + } + }, + "child_page": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "title": { + "type": "string" + } + } + }, + "child_database": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "title": { + "type": "string" + } + } + }, + "code": { + "type": "object", + "properties": { + "caption": { + "type": "array", "items": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "properties": { - "annotations": { - "additionalProperties": true, - "properties": { - "bold": { - "type": [ - "null", - "boolean" - ] - }, - "code": { - "type": [ - "null", - "boolean" - ] - }, - "color": { - "type": [ - "null", - "string" - ] - }, - "italic": { - "type": [ - "null", - "boolean" - ] - }, - "strikethrough": { - "type": [ - "null", - "boolean" - ] - }, - "underline": { - "type": [ - "null", - "boolean" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "href": { + "type": { "type": [ "null", "string" ] }, - "plain_text": { + "text": { "type": [ "null", - "string" - ] - }, - "rich_text": { + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -2725,10 +3164,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -2736,98 +3181,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": "object" - }, - "child_database": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "title": { - "type": "string" - } - }, - "type": "object" - }, - "child_page": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "title": { - "type": "string" - } - }, - "type": "object" - }, - "code": { - "properties": { - "color": { - "type": "string" - }, - "language": { - "type": "string" - }, - "text": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -2836,40 +3248,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -2877,14 +3291,27 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "rich_text": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -2894,10 +3321,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -2905,93 +3338,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": "object" - }, - "created_time": { - "type": "string" - }, - "embed": { - "properties": { - "url": { - "type": "string" - } - }, - "type": "object" - }, - "equation": { - "properties": { - "expression": { - "type": "string" - } - }, - "type": "object" - }, - "file": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "caption": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -3000,40 +3405,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -3041,14 +3448,147 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "language": { + "enum": [ + "abap", + "arduino", + "bash", + "basic", + "c", + "clojure", + "coffeescript", + "c++", + "c#", + "css", + "dart", + "diff", + "docker", + "elixir", + "elm", + "erlang", + "flow", + "fortran", + "f#", + "gherkin", + "glsl", + "go", + "graphql", + "groovy", + "haskell", + "html", + "java", + "javascript", + "json", + "julia", + "kotlin", + "latex", + "less", + "lisp", + "livescript", + "lua", + "makefile", + "markdown", + "markup", + "matlab", + "mermaid", + "nix", + "objective-c", + "ocaml", + "pascal", + "perl", + "php", + "plain text", + "powershell", + "prolog", + "protobuf", + "python", + "r", + "reason", + "ruby", + "rust", + "sass", + "scala", + "scheme", + "scss", + "shell", + "sql", + "swift", + "typescript", + "vb.net", + "verilog", + "vhdl", + "visual basic", + "webassembly", + "xml", + "yaml", + "java/c/c++/c#" + ] + } + } + }, + "column": { + "type": "object" + }, + "column_list": { + "type": "object" + }, + "divider": { + "type": "object" + }, + "embed": { + "type": "object", + "properties": { + "url": { + "type": "string" + } + } + }, + "equation": { + "type": "object", + "properties": { + "expression": { + "type": "string" + } + } + }, + "file": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "file", + "external" + ] + }, + "caption": { + "type": [ + "null", + "array" + ], + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -3058,10 +3598,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -3069,119 +3615,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": [ - "null", - "array" - ] - }, - "external": { - "additionalProperties": true, - "properties": { - "url": { - "type": "string" - } - }, - "type": [ - "null", - "object" - ] - }, - "file": { - "additionalProperties": true, - "properties": { - "expiry_time": { - "type": "string" - }, - "url": { - "type": "string" - } - }, - "type": [ - "null", - "object" - ] - }, - "type": { - "type": "string" - } - }, - "type": [ - "null", - "object" - ] - }, - "has_children": { - "type": [ - "null", - "boolean" - ] - }, - "heading_1": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "color": { - "type": "string" - }, - "text": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -3190,40 +3682,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -3231,14 +3725,68 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "external": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + } + }, + "file": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + }, + "expiry_time": { + "type": "string", + "format": "date-time" + } + } + } + } + }, + "heading_1": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "color": { + "type": "string" + }, + "rich_text": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -3248,10 +3796,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -3259,80 +3813,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": [ - "null", - "object" - ] - }, - "heading_2": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "color": { - "type": "string" - }, - "text": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -3341,40 +3880,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -3382,14 +3923,43 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "is_toggleable": { + "type": "boolean" + } + } + }, + "heading_2": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "color": { + "type": "string" + }, + "rich_text": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -3399,10 +3969,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -3410,80 +3986,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": [ - "null", - "object" - ] - }, - "heading_3": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "color": { - "type": "string" - }, - "text": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -3492,40 +4053,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -3533,14 +4096,43 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "is_toggleable": { + "type": "boolean" + } + } + }, + "heading_3": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "color": { + "type": "string" + }, + "rich_text": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -3550,10 +4142,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -3561,80 +4159,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": [ - "null", - "object" - ] - }, - "id": { - "type": "string" - }, - "image": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "caption": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -3643,40 +4226,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -3684,14 +4269,49 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "is_toggleable": { + "type": "boolean" + } + } + }, + "image": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "file", + "external" + ] + }, + "caption": { + "type": [ + "null", + "array" + ], + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -3701,10 +4321,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -3712,179 +4338,194 @@ "string" ] } - }, + } + } + } + }, + "mention": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { "type": [ "null", - "object" + "string" ] + }, + "info": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + } + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "expression": { "type": [ "null", "string" ] + } + } + }, + "annotations": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] }, - "link": { - "additionalProperties": true, - "properties": { - "type": { - "type": "string" - }, - "url": { - "type": [ - "null", - "string" - ] - } - }, + "color": { "type": [ "null", - "object" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, - "type": { + "href": { "type": [ "null", "string" ] } - }, - "type": "object" - }, - "type": [ - "null", - "array" - ] + } + } }, "external": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "url": { "type": "string" } - }, + } + }, + "file": { "type": [ "null", "object" - ] - }, - "file": { + ], "additionalProperties": true, "properties": { - "expiry_time": { - "type": "string" - }, "url": { "type": "string" + }, + "expiry_time": { + "type": "string", + "format": "date-time" } - }, - "type": [ - "null", - "object" - ] + } + } + } + }, + "link_preview": { + "type": "object", + "properties": { + "url": { + "type": "string" + } + } + }, + "link_to_page": { + "type": "object", + "properties": { + "page_id": { + "type": "string" }, "type": { "type": "string" } - }, - "type": [ - "null", - "object" - ] - }, - "last_edited_time": { - "type": "string" + } }, "numbered_list_item": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "additionalProperties": true, "properties": { - "children": { - "items": { - "type": "object" - }, - "type": "array" - }, "color": { "type": "string" }, - "text": { + "rich_text": { + "type": "array", "items": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "properties": { - "annotations": { - "additionalProperties": true, - "properties": { - "bold": { - "type": [ - "null", - "boolean" - ] - }, - "code": { - "type": [ - "null", - "boolean" - ] - }, - "color": { - "type": [ - "null", - "string" - ] - }, - "italic": { - "type": [ - "null", - "boolean" - ] - }, - "strikethrough": { - "type": [ - "null", - "boolean" - ] - }, - "underline": { - "type": [ - "null", - "boolean" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "href": { + "type": { "type": [ "null", "string" ] }, - "plain_text": { + "text": { "type": [ "null", - "string" - ] - }, - "rich_text": { + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -3894,10 +4535,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -3905,85 +4552,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": "object" - }, - "object": { - "type": "string" - }, - "paragraph": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "children": { - "items": { - "type": "object" - }, - "type": "array" - }, - "color": { - "type": "string" - }, - "rich_text": { - "items": { - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -3992,40 +4619,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -4033,14 +4662,43 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "children": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "paragraph": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "color": { + "type": "string" + }, + "rich_text": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -4050,10 +4708,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -4061,67 +4725,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - }, - "text": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -4130,40 +4792,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -4171,14 +4835,52 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "children": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "pdf": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "file", + "external" + ] + }, + "caption": { + "type": [ + "null", + "array" + ], + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -4188,10 +4890,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -4199,74 +4907,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": "object" - }, - "pdf": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "caption": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -4275,40 +4974,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -4316,14 +5017,65 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "external": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + } + } + }, + "file": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "url": { + "type": "string" + }, + "expiry_time": { + "type": "string", + "format": "date-time" + } + } + } + } + }, + "quote": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "color": { + "type": "string" + }, + "rich_text": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -4333,10 +5085,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -4344,176 +5102,376 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, + } + } + } + }, + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "expression": { "type": [ "null", - "object" + "string" ] } - }, + } + }, + "annotations": { "type": [ "null", "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + } + } + }, + "plain_text": { + "type": [ + "null", + "string" ] }, - "type": { + "href": { "type": [ "null", "string" ] } - }, - "type": "object" - }, - "type": [ - "null", - "array" - ] - }, - "external": { - "additionalProperties": true, - "properties": { - "url": { - "type": "string" } - }, + } + }, + "children": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "synced_block": { + "type": "object", + "properties": { + "synced_from": { "type": [ "null", "object" - ] - }, - "file": { - "additionalProperties": true, + ], "properties": { - "expiry_time": { - "type": "string" + "type": { + "type": "string", + "enum": [ + "block_id" + ] }, - "url": { + "block_id": { "type": "string" } - }, + } + }, + "children": { "type": [ "null", - "object" - ] - }, - "type": { - "type": "string" + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + } } - }, - "type": [ - "null", - "object" - ] + } }, - "quote": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, + "table": { + "type": "object", "properties": { - "children": { - "items": { - "type": "object" - }, - "type": "array" + "table_width": { + "type": "integer" + }, + "has_column_header": { + "type": "boolean" }, + "has_row_header": { + "type": "boolean" + } + } + }, + "table_of_contents": { + "type": "object", + "properties": { "color": { "type": "string" - }, - "text": { + } + } + }, + "table_row": { + "type": "object", + "properties": { + "cells": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "array" + ], + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "text": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "content": { + "type": [ + "null", + "string" + ] + }, + "link": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "url" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "mention": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "info": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } + }, + "annotations": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { + "type": [ + "null", + "boolean" + ] + }, + "code": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": [ + "null", + "string" + ] + } + } + }, + "plain_text": { + "type": [ + "null", + "string" + ] + }, + "href": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + }, + "template": { + "type": "object", + "properties": { + "rich_text": { + "type": "array", "items": { "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "properties": { - "annotations": { - "additionalProperties": true, - "properties": { - "bold": { - "type": [ - "null", - "boolean" - ] - }, - "code": { - "type": [ - "null", - "boolean" - ] - }, - "color": { - "type": [ - "null", - "string" - ] - }, - "italic": { - "type": [ - "null", - "boolean" - ] - }, - "strikethrough": { - "type": [ - "null", - "boolean" - ] - }, - "underline": { - "type": [ - "null", - "boolean" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "href": { + "type": { "type": [ "null", "string" ] }, - "plain_text": { + "text": { "type": [ "null", - "string" - ] - }, - "rich_text": { + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -4523,10 +5481,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -4534,84 +5498,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": "object" - }, - "to_do": { - "properties": { - "checked": { - "type": [ - "null", - "boolean" - ] - }, - "children": { - "items": { - "type": "object" - }, - "type": "array" - }, - "text": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -4620,40 +5565,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -4661,14 +5608,32 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + } + } + }, + "to_do": { + "type": "object", + "properties": { + "rich_text": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -4678,10 +5643,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -4689,83 +5660,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": "object" - }, - "toggle": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "children": { - "items": { - "type": "object" - }, - "type": "array" - }, - "color": { - "type": "string" - }, - "text": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -4774,40 +5727,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -4815,14 +5770,52 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "checked": { + "type": [ + "null", + "boolean" + ] + }, + "color": { + "type": "string" + }, + "children": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "toggle": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "color": { + "type": "string" + }, + "rich_text": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -4832,10 +5825,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -4843,77 +5842,65 @@ "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "mention": { "type": [ "null", "object" - ] - }, - "text": { + ], "additionalProperties": true, "properties": { - "content": { + "type": { "type": [ "null", "string" ] }, - "link": { - "additionalProperties": true, + "info": { + "type": [ + "null", + "object" + ], "properties": { - "type": { - "type": "string" + "id": { + "type": [ + "null", + "string" + ] }, - "url": { + "object": { "type": [ "null", "string" ] } - }, - "type": [ - "null", - "object" - ] + } } - }, + } + }, + "equation": { "type": [ "null", "object" - ] + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } }, - "type": { + "annotations": { "type": [ "null", - "string" - ] - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": "object" - }, - "type": { - "type": "string" - }, - "vidoe": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": true, - "properties": { - "caption": { - "items": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "annotations": { + "object" + ], "additionalProperties": true, "properties": { "bold": { @@ -4922,40 +5909,42 @@ "boolean" ] }, - "code": { + "italic": { "type": [ "null", "boolean" ] }, - "color": { + "strikethrough": { "type": [ "null", - "string" + "boolean" ] }, - "italic": { + "underline": { "type": [ "null", "boolean" ] }, - "strikethrough": { + "code": { "type": [ "null", "boolean" ] }, - "underline": { + "color": { "type": [ "null", - "boolean" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, "href": { @@ -4963,14 +5952,52 @@ "null", "string" ] - }, - "plain_text": { + } + } + } + }, + "children": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "video": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "enum": [ + "file", + "external" + ] + }, + "caption": { + "type": [ + "null", + "array" + ], + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { "type": [ "null", "string" ] }, - "rich_text": { + "text": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "content": { @@ -4980,10 +6007,16 @@ ] }, "link": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "type": { - "type": "string" + "enum": [ + "url" + ] }, "url": { "type": [ @@ -4991,106 +6024,420 @@ "string" ] } - }, + } + } + } + }, + "mention": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "info": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "equation": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "expression": { + "type": [ + "null", + "string" + ] + } + } + }, + "annotations": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "bold": { + "type": [ + "null", + "boolean" + ] + }, + "italic": { + "type": [ + "null", + "boolean" + ] + }, + "strikethrough": { + "type": [ + "null", + "boolean" + ] + }, + "underline": { "type": [ "null", - "object" + "boolean" ] - } - }, - "type": [ - "null", - "object" - ] - }, - "text": { - "additionalProperties": true, - "properties": { - "content": { + }, + "code": { "type": [ "null", - "string" + "boolean" ] }, - "link": { - "additionalProperties": true, - "properties": { - "type": { - "type": "string" - }, - "url": { - "type": [ - "null", - "string" - ] - } - }, + "color": { "type": [ "null", - "object" + "string" ] } - }, + } + }, + "plain_text": { "type": [ "null", - "object" + "string" ] }, - "type": { + "href": { "type": [ "null", "string" ] } - }, - "type": "object" - }, - "type": [ - "null", - "array" - ] + } + } }, "external": { + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "url": { "type": "string" } - }, + } + }, + "file": { "type": [ "null", "object" - ] - }, - "file": { + ], "additionalProperties": true, "properties": { - "expiry_time": { - "type": "string" - }, "url": { "type": "string" + }, + "expiry_time": { + "type": "string", + "format": "date-time" } - }, + } + } + } + }, + "unsupported": { + "type": "object" + }, + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "required": [ + "row_id" + ] + } + }, + "x-infer-schema": true + }, + "key": [ + "/id" + ] + }, + { + "recommendedName": "comments", + "resourceConfig": { + "stream": "comments", + "syncMode": "incremental", + "cursorField": [ + "page_last_edited_time" + ] + }, + "documentSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": [ + "comment" + ] + }, + "id": { + "type": "string" + }, + "parent": { + "type": "object", + "properties": { + "type": { + "enum": [ + "page_id" + ] + }, + "page_id": { + "type": "string" + } + } + }, + "discussion_id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "last_edited_time": { + "type": "string", + "format": "date-time" + }, + "page_last_edited_time": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": [ + "user" + ] + }, + "id": { + "type": "string" + }, + "name": { "type": [ "null", - "object" + "string" + ] + }, + "avatar_url": { + "type": [ + "null", + "string" ] }, "type": { - "type": "string" + "enum": [ + "person", + "bot" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "email": { + "type": [ + "null", + "string" + ] + } + } + }, + "bot": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "owner": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "info": { + "type": [ + "object", + "boolean" + ], + "properties": { + "avatar_url": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "person": { + "type": [ + "null", + "object" + ], + "properties": { + "email": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "workspace": { + "type": [ + "null", + "boolean" + ] + } + } + }, + "workspace_name": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "rich_text": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "text": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "link": { + "type": [ + "null", + "object" + ] + } + } + }, + "annotations": { + "type": "object", + "properties": { + "bold": { + "type": "boolean" + }, + "italic": { + "type": "boolean" + }, + "strikethrough": { + "type": "boolean" + }, + "underline": { + "type": "boolean" + }, + "code": { + "type": "boolean" + }, + "color": { + "type": "string" + } + } + }, + "plain_text": { + "type": "string" + }, + "href": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" } }, - "type": [ - "null", - "object" + "required": [ + "row_id" ] } }, - "required": [ - "id" - ], - "type": "object" + "x-infer-schema": true }, "key": [ "/id" diff --git a/source-notion/tests/snapshots/snapshots__spec__capture.stdout.json b/source-notion/tests/snapshots/snapshots__spec__capture.stdout.json index b7ab8f78b8..42bc560657 100644 --- a/source-notion/tests/snapshots/snapshots__spec__capture.stdout.json +++ b/source-notion/tests/snapshots/snapshots__spec__capture.stdout.json @@ -3,137 +3,137 @@ "protocol": 3032023, "configSchema": { "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Notion Source Spec", + "type": "object", + "required": [ + "credentials" + ], "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format YYYY-MM-DDTHH:MM:SS.000Z. During incremental sync, any data generated before this date will not be replicated. If left blank, the start date will be set to 2 years before the present date.", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:MM:SS.000Z", + "examples": [ + "2020-11-16T00:00:00.000Z" + ], + "type": "string", + "format": "date-time" + }, "credentials": { - "description": "Pick an authentication method.", + "title": "Authentication Method", + "description": "Choose either OAuth or Access Token.", "discriminator": { "propertyName": "auth_type" }, + "type": "object", + "order": 1, "oneOf": [ { + "type": "object", + "title": "OAuth2.0", + "x-oauth2-provider": "notion", + "required": [ + "auth_type", + "client_id", + "client_secret", + "access_token" + ], "properties": { - "access_token": { - "airbyte_secret": true, - "description": "Access Token is a token you received by complete the OauthWebFlow of Notion.", - "title": "Access Token", - "type": "string" - }, "auth_type": { - "const": "OAuth2.0", - "default": "OAuth2.0", - "type": "string" + "type": "string", + "const": "OAuth2.0" }, "client_id": { - "airbyte_secret": true, - "description": "The ClientID of your Notion integration.", "title": "Client ID", - "type": "string" + "type": "string", + "description": "The Client ID of your Notion integration.", + "airbyte_secret": true }, "client_secret": { - "airbyte_secret": true, - "description": "The ClientSecret of your Notion integration.", "title": "Client Secret", - "type": "string" + "type": "string", + "description": "The Client Secret of your Notion integration.", + "airbyte_secret": true + }, + "access_token": { + "title": "Access Token", + "type": "string", + "description": "The Access Token received by completing the OAuth flow for your Notion integration.", + "airbyte_secret": true } - }, + } + }, + { + "type": "object", + "title": "Access Token", "required": [ "auth_type", - "client_id", - "client_secret", - "access_token" + "token" ], - "title": "OAuth2.0", - "type": "object", - "x-oauth2-provider": "notion" - }, - { "properties": { "auth_type": { - "const": "token", - "default": "token", - "type": "string" + "type": "string", + "const": "token" }, "token": { - "airbyte_secret": true, - "description": "Notion API access token, see the docs for more information: https://go.estuary.dev/u5BKFR", "title": "Access Token", - "type": "string" + "description": "The Access Token for your private Notion integration. See the docs for more information: https://go.estuary.dev/u5BKFR", + "type": "string", + "airbyte_secret": true } - }, - "required": [ - "auth_type", - "token" - ], - "title": "Access Token", - "type": "object" + } } - ], - "order": 1, - "title": "Authenticate using", - "type": "object" - }, - "start_date": { - "default": "2023-04-17T00:00:00.000Z", - "description": "UTC date and time in the format 2017-01-25T00:00:00.000Z. Any data before this date will not be replicated.", - "examples": [ - "2020-11-16T00:00:00.000Z" - ], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", - "title": "Start Date", - "type": "string" + ] } - }, - "required": [ - "start_date", - "credentials" - ], - "title": "Notion Source Spec", - "type": "object" + } }, "resourceConfigSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ResourceSpec", + "title": "ResourceConfig", + "description": "ResourceConfig encodes a configured resource stream", "type": "object", - "required": [ - "stream", - "syncMode" - ], "properties": { - "cursorField": { - "type": [ - "array", - "null" - ], - "items": { - "type": "string" - } - }, - "namespace": { - "type": [ - "string", - "null" - ] - }, "stream": { + "title": "Stream", + "description": "Name of this stream", "type": "string" }, "syncMode": { - "type": "string", + "title": "Sync Mode", + "description": "Sync this resource incrementally, or fully refresh it every run", "enum": [ - "incremental", - "full_refresh" - ] + "full_refresh", + "incremental" + ], + "type": "string" + }, + "namespace": { + "title": "Namespace", + "description": "Enclosing schema namespace of this resource", + "type": "string" + }, + "cursorField": { + "title": "Cursor Field", + "type": "array", + "items": { + "type": "string" + } } - } + }, + "required": [ + "stream", + "syncMode" + ], + "additionalProperties": false }, - "documentationUrl": "https://go.estuary.dev/9iXgpB", + "documentationUrl": "https://go.estuary.dev/source-notion", "oauth2": { "provider": "notion", "authUrlTemplate": "https://api.notion.com/v1/oauth/authorize?client_id={{#urlencode}}{{{ client_id }}}{{/urlencode}}&redirect_uri={{#urlencode}}{{{ redirect_uri }}}{{/urlencode}}&response_type=code&owner=user&state={{#urlencode}}{{{ state }}}{{/urlencode}}", "accessTokenUrlTemplate": "https://api.notion.com/v1/oauth/token", "accessTokenBody": "{\"grant_type\": \"authorization_code\", \"redirect_uri\": \"{{{ redirect_uri }}}\", \"code\": \"{{{ code }}}\"}", "accessTokenHeaders": { - "Authorization": "Basic {{#basicauth}}{{{ client_id }}}:{{{client_secret }}}{{/basicauth}}", + "Authorization": "Basic {{#basicauth}}{{{ client_id }}}:{{{ client_secret }}}{{/basicauth}}", "Content-Type": "application/json" }, "accessTokenResponseMap": { From a19e1859f83bbe3c702d41f03b06cfedbcb0afd9 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Fri, 22 Mar 2024 15:22:28 -0400 Subject: [PATCH 52/96] source-notion: clean up imported code and adapt unit tests --- source-notion/.dockerignore | 7 - source-notion/README.md | 91 -- source-notion/acceptance-test-config.yml | 39 - source-notion/bootstrap.md | 32 - source-notion/icon.svg | 1 - source-notion/integration_tests/__init__.py | 3 - .../integration_tests/abnormal_state.json | 35 - source-notion/integration_tests/acceptance.py | 14 - source-notion/integration_tests/catalog.json | 38 - .../integration_tests/configured_catalog.json | 71 -- .../integration_tests/expected_records.jsonl | 15 - .../incremental_catalog.json | 46 - .../integration_tests/invalid_config.json | 9 - .../integration_tests/sample_config.json | 7 - .../sample_config_oauth.json | 9 - .../integration_tests/sample_state.json | 46 - source-notion/main.py | 8 - source-notion/metadata.yaml | 49 - source-notion/poetry.lock | 1110 +++++++++++++++-- source-notion/pyproject.toml | 40 +- .../test_incremental_streams.py | 0 .../{unit_tests => tests}/test_source.py | 0 .../{unit_tests => tests}/test_streams.py | 0 source-notion/unit_tests/__init__.py | 3 - 24 files changed, 1011 insertions(+), 662 deletions(-) delete mode 100644 source-notion/.dockerignore delete mode 100644 source-notion/README.md delete mode 100644 source-notion/acceptance-test-config.yml delete mode 100644 source-notion/bootstrap.md delete mode 100644 source-notion/icon.svg delete mode 100644 source-notion/integration_tests/__init__.py delete mode 100644 source-notion/integration_tests/abnormal_state.json delete mode 100644 source-notion/integration_tests/acceptance.py delete mode 100644 source-notion/integration_tests/catalog.json delete mode 100644 source-notion/integration_tests/configured_catalog.json delete mode 100644 source-notion/integration_tests/expected_records.jsonl delete mode 100644 source-notion/integration_tests/incremental_catalog.json delete mode 100644 source-notion/integration_tests/invalid_config.json delete mode 100644 source-notion/integration_tests/sample_config.json delete mode 100644 source-notion/integration_tests/sample_config_oauth.json delete mode 100644 source-notion/integration_tests/sample_state.json delete mode 100644 source-notion/main.py delete mode 100644 source-notion/metadata.yaml rename source-notion/{unit_tests => tests}/test_incremental_streams.py (100%) rename source-notion/{unit_tests => tests}/test_source.py (100%) rename source-notion/{unit_tests => tests}/test_streams.py (100%) delete mode 100644 source-notion/unit_tests/__init__.py diff --git a/source-notion/.dockerignore b/source-notion/.dockerignore deleted file mode 100644 index 955cfbdc08..0000000000 --- a/source-notion/.dockerignore +++ /dev/null @@ -1,7 +0,0 @@ -* -!Dockerfile -!Dockerfile.test -!main.py -!source_notion -!setup.py -!secrets diff --git a/source-notion/README.md b/source-notion/README.md deleted file mode 100644 index 13b0ba9f2b..0000000000 --- a/source-notion/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Notion source connector - - -This is the repository for the Notion source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/notion). - -## Local development - -### Prerequisites -* Python (~=3.9) -* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation) - - -### Installing the connector -From this connector directory, run: -```bash -poetry install --with dev -``` - - -### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/notion) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_notion/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. - - -### Locally running the connector -``` -poetry run source-notion spec -poetry run source-notion check --config secrets/config.json -poetry run source-notion discover --config secrets/config.json -poetry run source-notion read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Running unit tests -To run unit tests locally, from the connector directory run: -``` -poetry run pytest unit_tests -``` - -### Building the docker image -1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) -2. Run the following command to build the docker image: -```bash -airbyte-ci connectors --name=source-notion build -``` - -An image will be available on your host with the tag `airbyte/source-notion:dev`. - - -### Running as a docker container -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-notion:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-notion:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-notion:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-notion:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` - -### Running our CI test suite -You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): -```bash -airbyte-ci connectors --name=source-notion test -``` - -### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - -### Dependency Management -All of your dependencies should be managed via Poetry. -To add a new dependency, run: -```bash -poetry add -``` - -Please commit the changes to `pyproject.toml` and `poetry.lock` files. - -## Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-notion test` -2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): - - bump the `dockerImageTag` value in in `metadata.yaml` - - bump the `version` value in `pyproject.toml` -3. Make sure the `metadata.yaml` content is up to date. -4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/notion.md`). -5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). -6. Pat yourself on the back for being an awesome contributor. -7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/source-notion/acceptance-test-config.yml b/source-notion/acceptance-test-config.yml deleted file mode 100644 index ab14b89dc5..0000000000 --- a/source-notion/acceptance-test-config.yml +++ /dev/null @@ -1,39 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-notion:dev -test_strictness_level: high -acceptance_tests: - spec: - tests: - - spec_path: "source_notion/spec.json" - connection: - tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - # 2.0.8 introduces a fix to _blocks.properties.table_row.cells, - # which was incorrectly added in 2.0.0 as an object array instead of an array of object arrays. - disable_for_version: 2.0.7 - basic_read: - tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: true - incremental: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - full_refresh: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/source-notion/bootstrap.md b/source-notion/bootstrap.md deleted file mode 100644 index 6d492039f8..0000000000 --- a/source-notion/bootstrap.md +++ /dev/null @@ -1,32 +0,0 @@ -# Notion - -## Overview - -Notion is an application that provides components such as notes, databases, kanban boards, wikis, calendars and reminders. Notion REST API allows a developer to retrieve pages, databases, blocks, and users on the Notion platform. - -## Endpoints - -Notion API consists of three endpoints which can be extracted data from: - -1. **User**: The User object represents a user in a Notion workspace. Users include guests, full workspace members, and bots. -2. **Block**: A block object represents content within Notion. Blocks can be text, lists, media, and more. Page and database is also a type of block. -3. **Search**: This endpoint is used to get list of pages and databases. - -## Quick Notes - -- Notion stores content in hierarchy, each node is called a 'block'. Block is a generic term which can be text, lists, media, even page and database are also block. - -- Due to this hierarchical structure, we use recursive request to get the full list of blocks. - -- Pages and databases can be extracted from the `Search` endpoint separately, so they are excluded from the block list request. - -- Airbyte CDK doesn't support recursive schema, so some elements of the block schema which can be recursive are replaced with empty objects. - -- Page and database must grant permission to the internal integration, otherwise API cannot extract data from them. See [https://developers.notion.com/docs/authorization#authorizing-internal-integrations](https://developers.notion.com/docs/authorization#authorizing-internal-integrations) - -- Rate limiting is a standard exponential backoff when a 429 HTTP status code returned. The rate limit for incoming requests is an average of 3 requests per second. Some bursts beyond the average rate are allowed. Notion API also has size limit, see [https://developers.notion.com/reference/errors#request-limits](https://developers.notion.com/reference/errors#request-limits) - -## API Reference - -The API reference documents: [https://developers.notion.com/reference/intro](https://developers.notion.com/reference) - diff --git a/source-notion/icon.svg b/source-notion/icon.svg deleted file mode 100644 index 8b34d594bd..0000000000 --- a/source-notion/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/source-notion/integration_tests/__init__.py b/source-notion/integration_tests/__init__.py deleted file mode 100644 index 46b7376756..0000000000 --- a/source-notion/integration_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/source-notion/integration_tests/abnormal_state.json b/source-notion/integration_tests/abnormal_state.json deleted file mode 100644 index b791f731fb..0000000000 --- a/source-notion/integration_tests/abnormal_state.json +++ /dev/null @@ -1,35 +0,0 @@ -[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "last_edited_time": "2099-10-10T04:40:00.000Z" - }, - "stream_descriptor": { - "name": "databases" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "last_edited_time": "2099-10-10T04:40:00.000Z" - }, - "stream_descriptor": { - "name": "pages" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "last_edited_time": "2099-10-10T04:00:00.000Z" - }, - "stream_descriptor": { - "name": "blocks" - } - } - } -] diff --git a/source-notion/integration_tests/acceptance.py b/source-notion/integration_tests/acceptance.py deleted file mode 100644 index 82823254d2..0000000000 --- a/source-notion/integration_tests/acceptance.py +++ /dev/null @@ -1,14 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pytest - -pytest_plugins = ("connector_acceptance_test.plugin",) - - -@pytest.fixture(scope="session", autouse=True) -def connector_setup(): - """This fixture is a placeholder for external resources that acceptance test might require.""" - yield diff --git a/source-notion/integration_tests/catalog.json b/source-notion/integration_tests/catalog.json deleted file mode 100644 index 3b76e23aa3..0000000000 --- a/source-notion/integration_tests/catalog.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "streams": [ - { - "name": "users", - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "json_schema": {} - }, - { - "name": "databases", - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": "last_edited_time", - "json_schema": {} - }, - { - "name": "pages", - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": "last_edited_time", - "json_schema": {} - }, - { - "name": "blocks", - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": "last_edited_time", - "json_schema": {} - }, - { - "name": "comments", - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": "page_last_edited_time", - "json_schema": {} - } - ] -} diff --git a/source-notion/integration_tests/configured_catalog.json b/source-notion/integration_tests/configured_catalog.json deleted file mode 100644 index d0fa294445..0000000000 --- a/source-notion/integration_tests/configured_catalog.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "users", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "default_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] - }, - { - "stream": { - "name": "databases", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_primary_key": [["id"]], - "default_cursor_field": ["last_edited_time"] - }, - "primary_key": [["id"]], - "cursor_field": ["last_edited_time"], - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "pages", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_primary_key": [["id"]], - "default_cursor_field": ["last_edited_time"] - }, - "primary_key": [["id"]], - "cursor_field": ["last_edited_time"], - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "blocks", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_primary_key": [["id"]], - "default_cursor_field": ["last_edited_time"] - }, - "primary_key": [["id"]], - "cursor_field": ["last_edited_time"], - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "comments", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_primary_key": [["id"]], - "default_cursor_field": ["page_last_edited_time"] - }, - "primary_key": [["id"]], - "cursor_field": ["page_last_edited_time"], - "sync_mode": "incremental", - "destination_sync_mode": "append" - } - ] -} diff --git a/source-notion/integration_tests/expected_records.jsonl b/source-notion/integration_tests/expected_records.jsonl deleted file mode 100644 index 9989f84792..0000000000 --- a/source-notion/integration_tests/expected_records.jsonl +++ /dev/null @@ -1,15 +0,0 @@ -{"stream": "users", "data": {"object": "user", "id": "5612c094-99ec-4ba3-ac7f-df8d84c8d6be", "name": "Sherif Nada", "avatar_url": "https://s3-us-west-2.amazonaws.com/public.notion-static.com/305f7efc-2862-4342-ba99-5023f3e34717/6246757.png", "type": "person", "person": {"email": "sherif@airbyte.io"}}, "emitted_at": 1697023279924} -{"stream": "users", "data": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}, "emitted_at": 1697023279925} -{"stream": "users", "data": {"object": "user", "id": "c1ff0160-b2af-497a-aab7-8b61e625e4e3", "name": "Gil Cho", "avatar_url": "https://lh3.googleusercontent.com/a/ALm5wu0ElXfvy3YfVUyRn-aB9EZy5AZ1ougHuNyCGmO2=s100", "type": "person", "person": {"email": "gil@airbyte.io"}}, "emitted_at": 1697023279925} -{"stream": "databases", "data": {"object": "database", "id": "b75d2e55-cc80-4afa-a273-c78178ac6b3f", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\ude4b"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Engineering Directory ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Engineering Directory ", "href": null}], "description": [{"type": "text", "text": {"content": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "href": null}, {"type": "text", "text": {"content": "Notion databases", "link": {"url": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Notion databases", "href": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}, {"type": "text", "text": {"content": ".", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": ".", "href": null}], "is_inline": false, "properties": [{"name": "Date Added", "value": {"id": "%2Fkv%22", "name": "Date Added", "type": "created_time", "created_time": {}}}, {"name": "Notes", "value": {"id": "mq%22D", "name": "Notes", "type": "rich_text", "rich_text": {}}}, {"name": "Person", "value": {"id": "uiZ%26", "name": "Person", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "block_id", "block_id": "b81f8caf-3ec4-4455-9a0b-25c2bd3b60cb"}, "url": "https://www.notion.so/b75d2e55cc804afaa273c78178ac6b3f", "public_url": null, "archived": false}, "emitted_at": 1708341487319} -{"stream": "databases", "data": {"object": "database", "id": "fbff7d4e-eca4-4432-91e6-ec64ba4b5a98", "cover": null, "icon": null, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Questions", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Questions", "href": null}], "description": [], "is_inline": true, "properties": [{"name": "Difficulty", "value": {"id": "'i6%2F", "name": "Difficulty", "type": "select", "select": {"options": [{"id": "f00068b9-7612-45da-91ad-1a7b1d259375", "name": "Easy", "color": "green", "description": null}, {"id": "8e244bfe-d4c7-48c5-9088-ffd6926b4ba0", "name": "Medium", "color": "yellow", "description": null}, {"id": "9ab57ef4-eab1-4b20-a502-047610b5c97d", "name": "Hard", "color": "red", "description": null}]}}}, {"name": "Skills", "value": {"id": "K%3AtR", "name": "Skills", "type": "multi_select", "multi_select": {"options": [{"id": "72f4d134-a773-48c1-ba3d-b529f55c6818", "name": "Front end", "color": "default", "description": null}, {"id": "c20f5d57-3e35-4b39-b556-05071203cc1a", "name": "Backend", "color": "default", "description": null}, {"id": "31d5735c-d6ba-4bd7-940f-bdcb36091c02", "name": "Architecture", "color": "default", "description": null}, {"id": "0398de54-af68-4c3a-9953-3788e8eaadbf", "name": "Algorithms", "color": "default", "description": null}, {"id": "df9dff09-7dea-4409-a10f-b5e2b546ad94", "name": "Data Structures", "color": "default", "description": null}]}}}, {"name": "Question Name", "value": {"id": "title", "name": "Question Name", "type": "title", "title": {}}}], "parent": {"type": "page_id", "page_id": "4999109d-1b7b-41a2-abb4-84f6b961ee74"}, "url": "https://www.notion.so/fbff7d4eeca4443291e6ec64ba4b5a98", "public_url": null, "archived": false}, "emitted_at": 1708341487319} -{"stream": "databases", "data": {"object": "database", "id": "9b1ce91e-a93a-437c-8c92-81083cd98540", "cover": null, "icon": {"type": "emoji", "emoji": "\u270f\ufe0f"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "last_edited_time": "2023-09-13T00:06:00.000Z", "title": [{"type": "text", "text": {"content": "Meeting Notes", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Meeting Notes", "href": null}], "description": [{"type": "text", "text": {"content": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "href": null}, {"type": "text", "text": {"content": "\u2193", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "\u2193", "href": null}, {"type": "text", "text": {"content": " Click ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " Click ", "href": null}, {"type": "text", "text": {"content": "List View", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": true, "color": "default"}, "plain_text": "List View", "href": null}, {"type": "text", "text": {"content": " to create and see other views, including a board organized by meeting type.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " to create and see other views, including a board organized by meeting type.", "href": null}], "is_inline": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "name": "Last Edited Time", "type": "last_edited_time", "last_edited_time": {}}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "name": "Created By", "type": "created_by", "created_by": {}}}, {"name": "Created", "value": {"id": "Ird4", "name": "Created", "type": "created_time", "created_time": {}}}, {"name": "Type", "value": {"id": "_%7B%5C7", "name": "Type", "type": "select", "select": {"options": [{"id": "3a8fd64c-899d-4c39-ba97-ac4f565d6e94", "name": "Post-mortem", "color": "red", "description": null}, {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green", "description": null}, {"id": "8ee247a9-cb60-430a-9ea6-d5c053253334", "name": "Weekly Sync", "color": "blue", "description": null}, {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow", "description": null}, {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange", "description": null}]}}}, {"name": "Participants", "value": {"id": "b%3AeA", "name": "Participants", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/9b1ce91ea93a437c8c9281083cd98540", "public_url": null, "archived": false}, "emitted_at": 1708341487319} -{"stream": "pages", "data": {"object": "page", "id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Daily Standup", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Daily Standup", "href": null}]}}], "url": "https://www.notion.so/Daily-Standup-39a69b4e7cc24f7aa656dd128f3ce855", "public_url": null}, "emitted_at": 1697023284463} -{"stream": "pages", "data": {"object": "page", "id": "621d3dc4-55fe-46ce-a3ff-83da06e5f9fb", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Sprint Planning ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Sprint Planning ", "href": null}]}}], "url": "https://www.notion.so/Sprint-Planning-621d3dc455fe46cea3ff83da06e5f9fb", "public_url": null}, "emitted_at": 1697023284465} -{"stream": "pages", "data": {"object": "page", "id": "6eb2dedc-8b88-486c-8648-d1878bafb106", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Ad Hoc Meeting", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Ad Hoc Meeting", "href": null}]}}], "url": "https://www.notion.so/Ad-Hoc-Meeting-6eb2dedc8b88486c8648d1878bafb106", "public_url": null}, "emitted_at": 1697023284465} -{"stream": "blocks", "data": {"object": "block", "id": "b54364a0-ff86-45ba-b78e-a32018446a3f", "parent": {"type": "page_id", "page_id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "callout", "callout": {"rich_text": [{"type": "text", "text": {"content": "Change the title to include the date.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Change the title to include the date.", "href": null}], "icon": {"type": "emoji", "emoji": "\ud83d\udca1"}, "color": "gray_background"}}, "emitted_at": 1697023288683} -{"stream": "blocks", "data": {"object": "block", "id": "c6608513-db08-4411-8ec6-e343580cbf84", "parent": {"type": "page_id", "page_id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"rich_text": [{"type": "text", "text": {"content": "What did we do yesterday?", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "What did we do yesterday?", "href": null}], "is_toggleable": false, "color": "default"}}, "emitted_at": 1697023288684} -{"stream": "blocks", "data": {"object": "block", "id": "ffb233aa-59da-4d04-9cc8-e6b767bd1a85", "parent": {"type": "page_id", "page_id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "bulleted_list_item", "bulleted_list_item": {"rich_text": [], "color": "default"}}, "emitted_at": 1697023288684} -{"stream": "comments", "data": {"object": "comment", "id": "e2dd2530-3ef1-4a27-83fb-3f16400b9838", "parent": {"type": "page_id", "page_id": "a55d276e-4bc2-4fcc-9fb3-e60b867c86e7"}, "discussion_id": "15e3cffe-3c9d-4ef2-87b1-86c46f85a205", "created_time": "2023-10-10T13:52:00.000Z", "last_edited_time": "2023-10-10T13:52:00.000Z", "created_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "rich_text": [{"type": "text", "text": {"content": "Gathered voices speak,\nIdeas flow, plans take shape,\nWeek's promise whispers.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Gathered voices speak,\nIdeas flow, plans take shape,\nWeek's promise whispers.", "href": null}], "page_last_edited_time": "2021-10-19T13:33:00.000Z"}, "emitted_at": 1697023326959} -{"stream": "comments", "data": {"object": "comment", "id": "e2087302-7eab-4e1d-a95b-472404fb51a2", "parent": {"type": "page_id", "page_id": "249f3796-7e81-47b0-9075-00ed2d06439d"}, "discussion_id": "be372b5d-2610-435e-981a-9be271874b8e", "created_time": "2023-09-12T20:55:00.000Z", "last_edited_time": "2023-09-12T20:55:00.000Z", "created_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "rich_text": [{"type": "text", "text": {"content": "Ladybug in flight; Red and black on petal\u2019s edge; Spring\u2019s tiny delight.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Ladybug in flight; Red and black on petal\u2019s edge; Spring\u2019s tiny delight.", "href": null}], "page_last_edited_time": "2021-10-19T13:33:00.000Z"}, "emitted_at": 1697023328874} -{"stream": "comments", "data": {"object": "comment", "id": "3f7ce236-fbc5-4b10-bb75-a0835c00aff3", "parent": {"type": "page_id", "page_id": "29299296-ef3f-4aff-aef5-02d651a59be3"}, "discussion_id": "c30808ec-fabd-4e7b-947f-a36dbec5c1db", "created_time": "2023-09-12T20:56:00.000Z", "last_edited_time": "2023-09-12T20:56:00.000Z", "created_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "rich_text": [{"type": "text", "text": {"content": "APIs converge; Fixing gaps in code and docs; Two worlds now as one.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "APIs converge; Fixing gaps in code and docs; Two worlds now as one.", "href": null}], "page_last_edited_time": "2021-10-19T13:33:00.000Z"}, "emitted_at": 1697023329304} diff --git a/source-notion/integration_tests/incremental_catalog.json b/source-notion/integration_tests/incremental_catalog.json deleted file mode 100644 index 478309ad76..0000000000 --- a/source-notion/integration_tests/incremental_catalog.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "databases", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_primary_key": [["id"]], - "default_cursor_field": ["last_edited_time"] - }, - "primary_key": [["id"]], - "cursor_field": ["last_edited_time"], - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "pages", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_primary_key": [["id"]], - "default_cursor_field": ["last_edited_time"] - }, - "primary_key": [["id"]], - "cursor_field": ["last_edited_time"], - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "blocks", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_primary_key": [["id"]], - "default_cursor_field": ["last_edited_time"] - }, - "primary_key": [["id"]], - "cursor_field": ["last_edited_time"], - "sync_mode": "incremental", - "destination_sync_mode": "append" - } - ] -} diff --git a/source-notion/integration_tests/invalid_config.json b/source-notion/integration_tests/invalid_config.json deleted file mode 100644 index cad13a3770..0000000000 --- a/source-notion/integration_tests/invalid_config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "start_date": "2021-01-01T00:00:00.000Z", - "credentials": { - "auth_type": "OAuth2.0", - "client_id": "client_id", - "client_secret": "client_secret", - "access_token": "access_token" - } -} diff --git a/source-notion/integration_tests/sample_config.json b/source-notion/integration_tests/sample_config.json deleted file mode 100644 index 339023ce3b..0000000000 --- a/source-notion/integration_tests/sample_config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "start_date": "2021-01-01T00:00:00.000Z", - "credentials": { - "auth_type": "Token", - "access_token_": "" - } -} diff --git a/source-notion/integration_tests/sample_config_oauth.json b/source-notion/integration_tests/sample_config_oauth.json deleted file mode 100644 index 2aedd62665..0000000000 --- a/source-notion/integration_tests/sample_config_oauth.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "start_date": "2021-01-01T00:00:00.000Z", - "credentials": { - "auth_type": "OAuth", - "client_id": "", - "client_secret": "", - "access_token": "" - } -} diff --git a/source-notion/integration_tests/sample_state.json b/source-notion/integration_tests/sample_state.json deleted file mode 100644 index 0c49b5a9f4..0000000000 --- a/source-notion/integration_tests/sample_state.json +++ /dev/null @@ -1,46 +0,0 @@ -[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "last_edited_time": "2021-10-10T04:40:00.000Z" - }, - "stream_descriptor": { - "name": "databases" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "last_edited_time": "2021-10-10T04:40:00.000Z" - }, - "stream_descriptor": { - "name": "pages" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "last_edited_time": "2021-10-10T04:00:00.000Z" - }, - "stream_descriptor": { - "name": "blocks" - } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "last_edited_time": "2021-10-10T04:00:00.000Z" - }, - "stream_descriptor": { - "name": "comments" - } - } - } -] diff --git a/source-notion/main.py b/source-notion/main.py deleted file mode 100644 index 671d6cd692..0000000000 --- a/source-notion/main.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from source_notion.run import run - -if __name__ == "__main__": - run() diff --git a/source-notion/metadata.yaml b/source-notion/metadata.yaml deleted file mode 100644 index 3c6c34dd66..0000000000 --- a/source-notion/metadata.yaml +++ /dev/null @@ -1,49 +0,0 @@ -data: - ab_internal: - ql: 400 - sl: 200 - allowedHosts: - hosts: - - api.notion.com - connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c - connectorSubtype: api - connectorType: source - definitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 - dockerImageTag: 2.1.0 - dockerRepository: airbyte/source-notion - documentationUrl: https://docs.airbyte.com/integrations/sources/notion - githubIssueLabel: source-notion - icon: notion.svg - license: MIT - name: Notion - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-notion - registries: - cloud: - enabled: true - oss: - enabled: true - releaseStage: generally_available - releases: - breakingChanges: - 2.0.0: - message: - Version 2.0.0 introduces schema changes to multiple properties shared - by the blocks, databases and pages streams. These changes were introduced - to reflect updates to the Notion API. A full schema refresh and data reset - are required to upgrade to this version. - upgradeDeadline: "2023-11-09" - suggestedStreams: - streams: - - blocks - - databases - - pages - - users - supportLevel: certified - tags: - - language:python - - cdk:python -metadataSpecVersion: "1.0" diff --git a/source-notion/poetry.lock b/source-notion/poetry.lock index deaacddf79..e1780b82f4 100644 --- a/source-notion/poetry.lock +++ b/source-notion/poetry.lock @@ -1,14 +1,137 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "aiodns" +version = "3.1.1" +description = "Simple DNS resolver for asyncio" +optional = false +python-versions = "*" +files = [ + {file = "aiodns-3.1.1-py3-none-any.whl", hash = "sha256:a387b63da4ced6aad35b1dda2d09620ad608a1c7c0fb71efa07ebb4cd511928d"}, + {file = "aiodns-3.1.1.tar.gz", hash = "sha256:1073eac48185f7a4150cad7f96a5192d6911f12b4fb894de80a088508c9b3a99"}, +] + +[package.dependencies] +pycares = ">=4.0.0" + +[[package]] +name = "aiohttp" +version = "3.9.3" +description = "Async http client/server framework (asyncio)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, + {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, + {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, + {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, + {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, + {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, + {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, + {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, + {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, + {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, + {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, + {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, +] + +[package.dependencies] +aiosignal = ">=1.1.2" +attrs = ">=17.3.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns", "brotlicffi"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + +[package.dependencies] +frozenlist = ">=1.1.0" [[package]] name = "airbyte-cdk" -version = "0.52.7" +version = "0.52.10" description = "A framework for writing Airbyte Connectors." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte-cdk-0.52.7.tar.gz", hash = "sha256:73c55464ed57e030681fb4407613a5c0f07c519a4ba63aa9a4dd43d05cdf100b"}, - {file = "airbyte_cdk-0.52.7-py3-none-any.whl", hash = "sha256:f11665fc8f8dd2632d94e57f53991f7aaba8e9643a8ee7557f3040e40ea677ff"}, + {file = "airbyte-cdk-0.52.10.tar.gz", hash = "sha256:0daee950fe0d4453e6ceea2633090fc1d2144224e6f170b3c6cb4c6392811b47"}, + {file = "airbyte_cdk-0.52.10-py3-none-any.whl", hash = "sha256:366fd7bbbba317223edc1571d22b91c6f5bcff4ba65b3131e42f9b37e29932f4"}, ] [package.dependencies] @@ -50,16 +173,6 @@ files = [ [package.dependencies] pydantic = ">=1.9.2,<2.0.0" -[[package]] -name = "atomicwrites" -version = "1.4.1" -description = "Atomic file writes." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] - [[package]] name = "attrs" version = "23.2.0" @@ -103,13 +216,13 @@ files = [ [[package]] name = "cachetools" -version = "5.3.2" +version = "5.3.3" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, - {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, ] [[package]] @@ -125,8 +238,6 @@ files = [ [package.dependencies] attrs = ">=23.1.0" -exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} [package.extras] bson = ["pymongo (>=4.4.0)"] @@ -148,6 +259,70 @@ files = [ {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, ] +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -258,6 +433,37 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "debugpy" +version = "1.8.1" +description = "An implementation of the Debug Adapter Protocol for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "debugpy-1.8.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:3bda0f1e943d386cc7a0e71bfa59f4137909e2ed947fb3946c506e113000f741"}, + {file = "debugpy-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dda73bf69ea479c8577a0448f8c707691152e6c4de7f0c4dec5a4bc11dee516e"}, + {file = "debugpy-1.8.1-cp310-cp310-win32.whl", hash = "sha256:3a79c6f62adef994b2dbe9fc2cc9cc3864a23575b6e387339ab739873bea53d0"}, + {file = "debugpy-1.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:7eb7bd2b56ea3bedb009616d9e2f64aab8fc7000d481faec3cd26c98a964bcdd"}, + {file = "debugpy-1.8.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:016a9fcfc2c6b57f939673c874310d8581d51a0fe0858e7fac4e240c5eb743cb"}, + {file = "debugpy-1.8.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd97ed11a4c7f6d042d320ce03d83b20c3fb40da892f994bc041bbc415d7a099"}, + {file = "debugpy-1.8.1-cp311-cp311-win32.whl", hash = "sha256:0de56aba8249c28a300bdb0672a9b94785074eb82eb672db66c8144fff673146"}, + {file = "debugpy-1.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:1a9fe0829c2b854757b4fd0a338d93bc17249a3bf69ecf765c61d4c522bb92a8"}, + {file = "debugpy-1.8.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3ebb70ba1a6524d19fa7bb122f44b74170c447d5746a503e36adc244a20ac539"}, + {file = "debugpy-1.8.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2e658a9630f27534e63922ebf655a6ab60c370f4d2fc5c02a5b19baf4410ace"}, + {file = "debugpy-1.8.1-cp312-cp312-win32.whl", hash = "sha256:caad2846e21188797a1f17fc09c31b84c7c3c23baf2516fed5b40b378515bbf0"}, + {file = "debugpy-1.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:edcc9f58ec0fd121a25bc950d4578df47428d72e1a0d66c07403b04eb93bcf98"}, + {file = "debugpy-1.8.1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:7a3afa222f6fd3d9dfecd52729bc2e12c93e22a7491405a0ecbf9e1d32d45b39"}, + {file = "debugpy-1.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d915a18f0597ef685e88bb35e5d7ab968964b7befefe1aaea1eb5b2640b586c7"}, + {file = "debugpy-1.8.1-cp38-cp38-win32.whl", hash = "sha256:92116039b5500633cc8d44ecc187abe2dfa9b90f7a82bbf81d079fcdd506bae9"}, + {file = "debugpy-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:e38beb7992b5afd9d5244e96ad5fa9135e94993b0c551ceebf3fe1a5d9beb234"}, + {file = "debugpy-1.8.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:bfb20cb57486c8e4793d41996652e5a6a885b4d9175dd369045dad59eaacea42"}, + {file = "debugpy-1.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd3fdd3f67a7e576dd869c184c5dd71d9aaa36ded271939da352880c012e703"}, + {file = "debugpy-1.8.1-cp39-cp39-win32.whl", hash = "sha256:58911e8521ca0c785ac7a0539f1e77e0ce2df753f786188f382229278b4cdf23"}, + {file = "debugpy-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:6df9aa9599eb05ca179fb0b810282255202a66835c6efb1d112d21ecb830ddd3"}, + {file = "debugpy-1.8.1-py2.py3-none-any.whl", hash = "sha256:28acbe2241222b87e255260c76741e1fbf04fdc3b6d094fcf57b6c6f75ce1242"}, + {file = "debugpy-1.8.1.zip", hash = "sha256:f696d6be15be87aef621917585f9bb94b1dc9e8aced570db1b8a6fc14e8f9b42"}, +] + [[package]] name = "deprecated" version = "1.2.14" @@ -287,18 +493,24 @@ files = [ ] [[package]] -name = "exceptiongroup" -version = "1.2.0" -description = "Backport of PEP 654 (exception groups)" +name = "estuary-cdk" +version = "0.2.0" +description = "Estuary Connector Development Kit" optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, -] +python-versions = "^3.11" +files = [] +develop = true -[package.extras] -test = ["pytest (>=6)"] +[package.dependencies] +aiodns = "^3.1.1" +aiohttp = "^3.9.3" +orjson = "^3.9.15" +pydantic = ">1.10,<3" +xxhash = "^3.4.1" + +[package.source] +type = "directory" +url = "../estuary-cdk" [[package]] name = "freezegun" @@ -314,6 +526,92 @@ files = [ [package.dependencies] python-dateutil = ">=2.7" +[[package]] +name = "frozenlist" +version = "1.4.1" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = false +python-versions = ">=3.8" +files = [ + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc"}, + {file = "frozenlist-1.4.1-cp310-cp310-win32.whl", hash = "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1"}, + {file = "frozenlist-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"}, + {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"}, + {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8"}, + {file = "frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89"}, + {file = "frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7"}, + {file = "frozenlist-1.4.1-cp38-cp38-win32.whl", hash = "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497"}, + {file = "frozenlist-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6"}, + {file = "frozenlist-1.4.1-cp39-cp39-win32.whl", hash = "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932"}, + {file = "frozenlist-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0"}, + {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"}, + {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, +] + [[package]] name = "genson" version = "1.2.2" @@ -478,50 +776,330 @@ files = [ {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] +[[package]] +name = "multidict" +version = "6.0.5" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, +] + +[[package]] +name = "mypy" +version = "1.9.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8a67616990062232ee4c3952f41c779afac41405806042a8126fe96e098419f"}, + {file = "mypy-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d357423fa57a489e8c47b7c85dfb96698caba13d66e086b412298a1a0ea3b0ed"}, + {file = "mypy-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49c87c15aed320de9b438ae7b00c1ac91cd393c1b854c2ce538e2a72d55df150"}, + {file = "mypy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:48533cdd345c3c2e5ef48ba3b0d3880b257b423e7995dada04248725c6f77374"}, + {file = "mypy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:4d3dbd346cfec7cb98e6cbb6e0f3c23618af826316188d587d1c1bc34f0ede03"}, + {file = "mypy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:653265f9a2784db65bfca694d1edd23093ce49740b2244cde583aeb134c008f3"}, + {file = "mypy-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a3c007ff3ee90f69cf0a15cbcdf0995749569b86b6d2f327af01fd1b8aee9dc"}, + {file = "mypy-1.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2418488264eb41f69cc64a69a745fad4a8f86649af4b1041a4c64ee61fc61129"}, + {file = "mypy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:68edad3dc7d70f2f17ae4c6c1b9471a56138ca22722487eebacfd1eb5321d612"}, + {file = "mypy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:85ca5fcc24f0b4aeedc1d02f93707bccc04733f21d41c88334c5482219b1ccb3"}, + {file = "mypy-1.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aceb1db093b04db5cd390821464504111b8ec3e351eb85afd1433490163d60cd"}, + {file = "mypy-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0235391f1c6f6ce487b23b9dbd1327b4ec33bb93934aa986efe8a9563d9349e6"}, + {file = "mypy-1.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4d5ddc13421ba3e2e082a6c2d74c2ddb3979c39b582dacd53dd5d9431237185"}, + {file = "mypy-1.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:190da1ee69b427d7efa8aa0d5e5ccd67a4fb04038c380237a0d96829cb157913"}, + {file = "mypy-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe28657de3bfec596bbeef01cb219833ad9d38dd5393fc649f4b366840baefe6"}, + {file = "mypy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e54396d70be04b34f31d2edf3362c1edd023246c82f1730bbf8768c28db5361b"}, + {file = "mypy-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5e6061f44f2313b94f920e91b204ec600982961e07a17e0f6cd83371cb23f5c2"}, + {file = "mypy-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a10926e5473c5fc3da8abb04119a1f5811a236dc3a38d92015cb1e6ba4cb9e"}, + {file = "mypy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b685154e22e4e9199fc95f298661deea28aaede5ae16ccc8cbb1045e716b3e04"}, + {file = "mypy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:5d741d3fc7c4da608764073089e5f58ef6352bedc223ff58f2f038c2c4698a89"}, + {file = "mypy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:587ce887f75dd9700252a3abbc9c97bbe165a4a630597845c61279cf32dfbf02"}, + {file = "mypy-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f88566144752999351725ac623471661c9d1cd8caa0134ff98cceeea181789f4"}, + {file = "mypy-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61758fabd58ce4b0720ae1e2fea5cfd4431591d6d590b197775329264f86311d"}, + {file = "mypy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e49499be624dead83927e70c756970a0bc8240e9f769389cdf5714b0784ca6bf"}, + {file = "mypy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:571741dc4194b4f82d344b15e8837e8c5fcc462d66d076748142327626a1b6e9"}, + {file = "mypy-1.9.0-py3-none-any.whl", hash = "sha256:a260627a570559181a9ea5de61ac6297aa5af202f06fd7ab093ce74e7181e43e"}, + {file = "mypy-1.9.0.tar.gz", hash = "sha256:3cc5da0127e6a478cddd906068496a97a7618a21ce9b54bde5bf7e539c7af974"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "orjson" +version = "3.9.15" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.9.15-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:d61f7ce4727a9fa7680cd6f3986b0e2c732639f46a5e0156e550e35258aa313a"}, + {file = "orjson-3.9.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4feeb41882e8aa17634b589533baafdceb387e01e117b1ec65534ec724023d04"}, + {file = "orjson-3.9.15-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fbbeb3c9b2edb5fd044b2a070f127a0ac456ffd079cb82746fc84af01ef021a4"}, + {file = "orjson-3.9.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b66bcc5670e8a6b78f0313bcb74774c8291f6f8aeef10fe70e910b8040f3ab75"}, + {file = "orjson-3.9.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2973474811db7b35c30248d1129c64fd2bdf40d57d84beed2a9a379a6f57d0ab"}, + {file = "orjson-3.9.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fe41b6f72f52d3da4db524c8653e46243c8c92df826ab5ffaece2dba9cccd58"}, + {file = "orjson-3.9.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4228aace81781cc9d05a3ec3a6d2673a1ad0d8725b4e915f1089803e9efd2b99"}, + {file = "orjson-3.9.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6f7b65bfaf69493c73423ce9db66cfe9138b2f9ef62897486417a8fcb0a92bfe"}, + {file = "orjson-3.9.15-cp310-none-win32.whl", hash = "sha256:2d99e3c4c13a7b0fb3792cc04c2829c9db07838fb6973e578b85c1745e7d0ce7"}, + {file = "orjson-3.9.15-cp310-none-win_amd64.whl", hash = "sha256:b725da33e6e58e4a5d27958568484aa766e825e93aa20c26c91168be58e08cbb"}, + {file = "orjson-3.9.15-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:c8e8fe01e435005d4421f183038fc70ca85d2c1e490f51fb972db92af6e047c2"}, + {file = "orjson-3.9.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87f1097acb569dde17f246faa268759a71a2cb8c96dd392cd25c668b104cad2f"}, + {file = "orjson-3.9.15-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff0f9913d82e1d1fadbd976424c316fbc4d9c525c81d047bbdd16bd27dd98cfc"}, + {file = "orjson-3.9.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8055ec598605b0077e29652ccfe9372247474375e0e3f5775c91d9434e12d6b1"}, + {file = "orjson-3.9.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d6768a327ea1ba44c9114dba5fdda4a214bdb70129065cd0807eb5f010bfcbb5"}, + {file = "orjson-3.9.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12365576039b1a5a47df01aadb353b68223da413e2e7f98c02403061aad34bde"}, + {file = "orjson-3.9.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:71c6b009d431b3839d7c14c3af86788b3cfac41e969e3e1c22f8a6ea13139404"}, + {file = "orjson-3.9.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e18668f1bd39e69b7fed19fa7cd1cd110a121ec25439328b5c89934e6d30d357"}, + {file = "orjson-3.9.15-cp311-none-win32.whl", hash = "sha256:62482873e0289cf7313461009bf62ac8b2e54bc6f00c6fabcde785709231a5d7"}, + {file = "orjson-3.9.15-cp311-none-win_amd64.whl", hash = "sha256:b3d336ed75d17c7b1af233a6561cf421dee41d9204aa3cfcc6c9c65cd5bb69a8"}, + {file = "orjson-3.9.15-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:82425dd5c7bd3adfe4e94c78e27e2fa02971750c2b7ffba648b0f5d5cc016a73"}, + {file = "orjson-3.9.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c51378d4a8255b2e7c1e5cc430644f0939539deddfa77f6fac7b56a9784160a"}, + {file = "orjson-3.9.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6ae4e06be04dc00618247c4ae3f7c3e561d5bc19ab6941427f6d3722a0875ef7"}, + {file = "orjson-3.9.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bcef128f970bb63ecf9a65f7beafd9b55e3aaf0efc271a4154050fc15cdb386e"}, + {file = "orjson-3.9.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b72758f3ffc36ca566ba98a8e7f4f373b6c17c646ff8ad9b21ad10c29186f00d"}, + {file = "orjson-3.9.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c57bc7b946cf2efa67ac55766e41764b66d40cbd9489041e637c1304400494"}, + {file = "orjson-3.9.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:946c3a1ef25338e78107fba746f299f926db408d34553b4754e90a7de1d44068"}, + {file = "orjson-3.9.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2f256d03957075fcb5923410058982aea85455d035607486ccb847f095442bda"}, + {file = "orjson-3.9.15-cp312-none-win_amd64.whl", hash = "sha256:5bb399e1b49db120653a31463b4a7b27cf2fbfe60469546baf681d1b39f4edf2"}, + {file = "orjson-3.9.15-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b17f0f14a9c0ba55ff6279a922d1932e24b13fc218a3e968ecdbf791b3682b25"}, + {file = "orjson-3.9.15-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f6cbd8e6e446fb7e4ed5bac4661a29e43f38aeecbf60c4b900b825a353276a1"}, + {file = "orjson-3.9.15-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:76bc6356d07c1d9f4b782813094d0caf1703b729d876ab6a676f3aaa9a47e37c"}, + {file = "orjson-3.9.15-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fdfa97090e2d6f73dced247a2f2d8004ac6449df6568f30e7fa1a045767c69a6"}, + {file = "orjson-3.9.15-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7413070a3e927e4207d00bd65f42d1b780fb0d32d7b1d951f6dc6ade318e1b5a"}, + {file = "orjson-3.9.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9cf1596680ac1f01839dba32d496136bdd5d8ffb858c280fa82bbfeb173bdd40"}, + {file = "orjson-3.9.15-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:809d653c155e2cc4fd39ad69c08fdff7f4016c355ae4b88905219d3579e31eb7"}, + {file = "orjson-3.9.15-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:920fa5a0c5175ab14b9c78f6f820b75804fb4984423ee4c4f1e6d748f8b22bc1"}, + {file = "orjson-3.9.15-cp38-none-win32.whl", hash = "sha256:2b5c0f532905e60cf22a511120e3719b85d9c25d0e1c2a8abb20c4dede3b05a5"}, + {file = "orjson-3.9.15-cp38-none-win_amd64.whl", hash = "sha256:67384f588f7f8daf040114337d34a5188346e3fae6c38b6a19a2fe8c663a2f9b"}, + {file = "orjson-3.9.15-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6fc2fe4647927070df3d93f561d7e588a38865ea0040027662e3e541d592811e"}, + {file = "orjson-3.9.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34cbcd216e7af5270f2ffa63a963346845eb71e174ea530867b7443892d77180"}, + {file = "orjson-3.9.15-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f541587f5c558abd93cb0de491ce99a9ef8d1ae29dd6ab4dbb5a13281ae04cbd"}, + {file = "orjson-3.9.15-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92255879280ef9c3c0bcb327c5a1b8ed694c290d61a6a532458264f887f052cb"}, + {file = "orjson-3.9.15-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:05a1f57fb601c426635fcae9ddbe90dfc1ed42245eb4c75e4960440cac667262"}, + {file = "orjson-3.9.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ede0bde16cc6e9b96633df1631fbcd66491d1063667f260a4f2386a098393790"}, + {file = "orjson-3.9.15-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e88b97ef13910e5f87bcbc4dd7979a7de9ba8702b54d3204ac587e83639c0c2b"}, + {file = "orjson-3.9.15-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:57d5d8cf9c27f7ef6bc56a5925c7fbc76b61288ab674eb352c26ac780caa5b10"}, + {file = "orjson-3.9.15-cp39-none-win32.whl", hash = "sha256:001f4eb0ecd8e9ebd295722d0cbedf0748680fb9998d3993abaed2f40587257a"}, + {file = "orjson-3.9.15-cp39-none-win_amd64.whl", hash = "sha256:ea0b183a5fe6b2b45f3b854b0d19c4e932d6f5934ae1f723b07cf9560edd4ec7"}, + {file = "orjson-3.9.15.tar.gz", hash = "sha256:95cae920959d772f30ab36d3b25f83bb0f3be671e986c72ce22f8fa700dae061"}, +] + [[package]] name = "packaging" -version = "23.2" +version = "24.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, ] [[package]] name = "pendulum" -version = "2.1.2" +version = "3.0.0" description = "Python datetimes made easy" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.8" files = [ - {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, - {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, - {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, - {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, - {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, - {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, - {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, - {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, - {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, - {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, - {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, ] [package.dependencies] -python-dateutil = ">=2.6,<3.0" -pytzdata = ">=2020.1" +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] [[package]] name = "platformdirs" @@ -554,14 +1132,80 @@ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] [[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" +name = "pycares" +version = "4.4.0" +description = "Python interface for c-ares" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycares-4.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:24da119850841d16996713d9c3374ca28a21deee056d609fbbed29065d17e1f6"}, + {file = "pycares-4.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8f64cb58729689d4d0e78f0bfb4c25ce2f851d0274c0273ac751795c04b8798a"}, + {file = "pycares-4.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d33e2a1120887e89075f7f814ec144f66a6ce06a54f5722ccefc62fbeda83cff"}, + {file = "pycares-4.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c680fef1b502ee680f8f0b95a41af4ec2c234e50e16c0af5bbda31999d3584bd"}, + {file = "pycares-4.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fff16b09042ba077f7b8aa5868d1d22456f0002574d0ba43462b10a009331677"}, + {file = "pycares-4.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:229a1675eb33bc9afb1fc463e73ee334950ccc485bc83a43f6ae5839fb4d5fa3"}, + {file = "pycares-4.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3aebc73e5ad70464f998f77f2da2063aa617cbd8d3e8174dd7c5b4518f967153"}, + {file = "pycares-4.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6ef64649eba56448f65e26546d85c860709844d2fc22ef14d324fe0b27f761a9"}, + {file = "pycares-4.4.0-cp310-cp310-win32.whl", hash = "sha256:4afc2644423f4eef97857a9fd61be9758ce5e336b4b0bd3d591238bb4b8b03e0"}, + {file = "pycares-4.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:5ed4e04af4012f875b78219d34434a6d08a67175150ac1b79eb70ab585d4ba8c"}, + {file = "pycares-4.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bce8db2fc6f3174bd39b81405210b9b88d7b607d33e56a970c34a0c190da0490"}, + {file = "pycares-4.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a0303428d013ccf5c51de59c83f9127aba6200adb7fd4be57eddb432a1edd2a"}, + {file = "pycares-4.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afb91792f1556f97be7f7acb57dc7756d89c5a87bd8b90363a77dbf9ea653817"}, + {file = "pycares-4.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b61579cecf1f4d616e5ea31a6e423a16680ab0d3a24a2ffe7bb1d4ee162477ff"}, + {file = "pycares-4.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7af06968cbf6851566e806bf3e72825b0e6671832a2cbe840be1d2d65350710"}, + {file = "pycares-4.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ceb12974367b0a68a05d52f4162b29f575d241bd53de155efe632bf2c943c7f6"}, + {file = "pycares-4.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2eeec144bcf6a7b6f2d74d6e70cbba7886a84dd373c886f06cb137a07de4954c"}, + {file = "pycares-4.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e3a6f7cfdfd11eb5493d6d632e582408c8f3b429f295f8799c584c108b28db6f"}, + {file = "pycares-4.4.0-cp311-cp311-win32.whl", hash = "sha256:34736a2ffaa9c08ca9c707011a2d7b69074bbf82d645d8138bba771479b2362f"}, + {file = "pycares-4.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:eb66c30eb11e877976b7ead13632082a8621df648c408b8e15cdb91a452dd502"}, + {file = "pycares-4.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:fd644505a8cfd7f6584d33a9066d4e3d47700f050ef1490230c962de5dfb28c6"}, + {file = "pycares-4.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52084961262232ec04bd75f5043aed7e5d8d9695e542ff691dfef0110209f2d4"}, + {file = "pycares-4.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0c5368206057884cde18602580083aeaad9b860e2eac14fd253543158ce1e93"}, + {file = "pycares-4.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:112a4979c695b1c86f6782163d7dec58d57a3b9510536dcf4826550f9053dd9a"}, + {file = "pycares-4.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d186dafccdaa3409194c0f94db93c1a5d191145a275f19da6591f9499b8e7b8"}, + {file = "pycares-4.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:64965dc19c578a683ea73487a215a8897276224e004d50eeb21f0bc7a0b63c88"}, + {file = "pycares-4.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:ed2a38e34bec6f2586435f6ff0bc5fe11d14bebd7ed492cf739a424e81681540"}, + {file = "pycares-4.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:94d6962db81541eb0396d2f0dfcbb18cdb8c8b251d165efc2d974ae652c547d4"}, + {file = "pycares-4.4.0-cp312-cp312-win32.whl", hash = "sha256:1168a48a834813aa80f412be2df4abaf630528a58d15c704857448b20b1675c0"}, + {file = "pycares-4.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:db24c4e7fea4a052c6e869cbf387dd85d53b9736cfe1ef5d8d568d1ca925e977"}, + {file = "pycares-4.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:21a5a0468861ec7df7befa69050f952da13db5427ae41ffe4713bc96291d1d95"}, + {file = "pycares-4.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:22c00bf659a9fa44d7b405cf1cd69b68b9d37537899898d8cbe5dffa4016b273"}, + {file = "pycares-4.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23aa3993a352491a47fcf17867f61472f32f874df4adcbb486294bd9fbe8abee"}, + {file = "pycares-4.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:813d661cbe2e37d87da2d16b7110a6860e93ddb11735c6919c8a3545c7b9c8d8"}, + {file = "pycares-4.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:77cf5a2fd5583c670de41a7f4a7b46e5cbabe7180d8029f728571f4d2e864084"}, + {file = "pycares-4.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3eaa6681c0a3e3f3868c77aca14b7760fed35fdfda2fe587e15c701950e7bc69"}, + {file = "pycares-4.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ad58e284a658a8a6a84af2e0b62f2f961f303cedfe551854d7bd40c3cbb61912"}, + {file = "pycares-4.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bfb89ca9e3d0a9b5332deeb666b2ede9d3469107742158f4aeda5ce032d003f4"}, + {file = "pycares-4.4.0-cp38-cp38-win32.whl", hash = "sha256:f36bdc1562142e3695555d2f4ac0cb69af165eddcefa98efc1c79495b533481f"}, + {file = "pycares-4.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:902461a92b6a80fd5041a2ec5235680c7cc35e43615639ec2a40e63fca2dfb51"}, + {file = "pycares-4.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7bddc6adba8f699728f7fc1c9ce8cef359817ad78e2ed52b9502cb5f8dc7f741"}, + {file = "pycares-4.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cb49d5805cd347c404f928c5ae7c35e86ba0c58ffa701dbe905365e77ce7d641"}, + {file = "pycares-4.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56cf3349fa3a2e67ed387a7974c11d233734636fe19facfcda261b411af14d80"}, + {file = "pycares-4.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bf2eaa83a5987e48fa63302f0fe7ce3275cfda87b34d40fef9ce703fb3ac002"}, + {file = "pycares-4.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82bba2ab77eb5addbf9758d514d9bdef3c1bfe7d1649a47bd9a0d55a23ef478b"}, + {file = "pycares-4.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c6a8bde63106f162fca736e842a916853cad3c8d9d137e11c9ffa37efa818b02"}, + {file = "pycares-4.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f5f646eec041db6ffdbcaf3e0756fb92018f7af3266138c756bb09d2b5baadec"}, + {file = "pycares-4.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9dc04c54c6ea615210c1b9e803d0e2d2255f87a3d5d119b6482c8f0dfa15b26b"}, + {file = "pycares-4.4.0-cp39-cp39-win32.whl", hash = "sha256:97892cced5794d721fb4ff8765764aa4ea48fe8b2c3820677505b96b83d4ef47"}, + {file = "pycares-4.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:917f08f0b5d9324e9a34211e68d27447c552b50ab967044776bbab7e42a553a2"}, + {file = "pycares-4.4.0.tar.gz", hash = "sha256:f47579d508f2f56eddd16ce72045782ad3b1b3b678098699e2b6a1b30733e1c2"}, +] + +[package.dependencies] +cffi = ">=1.5.0" + +[package.extras] +idna = ["idna (>=2.1)"] + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] [[package]] @@ -659,70 +1303,70 @@ files = [ [[package]] name = "pytest" -version = "6.2.5" +version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, - {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, ] [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -toml = "*" [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-insta" +version = "0.3.0" +description = "A practical snapshot testing plugin for pytest" +optional = false +python-versions = ">=3.10,<4.0" +files = [ + {file = "pytest_insta-0.3.0-py3-none-any.whl", hash = "sha256:93a105e3850f2887b120a581923b10bb313d722e00d369377a1d91aa535df704"}, + {file = "pytest_insta-0.3.0.tar.gz", hash = "sha256:9e6e1c70a021f68ccc4643360b2c2f8326cf3befba85f942c1da17b9caf713f7"}, +] + +[package.dependencies] +pytest = ">=7.2.0,<9.0.0" +wrapt = ">=1.14.1,<2.0.0" [[package]] name = "pytest-mock" -version = "3.12.0" +version = "3.14.0" description = "Thin-wrapper around the mock package for easier use with pytest" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, - {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, ] [package.dependencies] -pytest = ">=5.0" +pytest = ">=6.2.5" [package.extras] dev = ["pre-commit", "pytest-asyncio", "tox"] [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] [package.dependencies] six = ">=1.5" -[[package]] -name = "pytzdata" -version = "2020.1" -description = "The Olson timezone database for Python." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, - {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, -] - [[package]] name = "pyyaml" version = "6.0.1" @@ -806,13 +1450,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-cache" -version = "1.1.1" +version = "1.2.0" description = "A persistent cache for python requests" optional = false -python-versions = ">=3.7,<4.0" +python-versions = ">=3.8" files = [ - {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, - {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, ] [package.dependencies] @@ -824,15 +1468,15 @@ url-normalize = ">=1.4" urllib3 = ">=1.25.5" [package.extras] -all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] bson = ["bson (>=0.5)"] -docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] json = ["ujson (>=5.4)"] mongodb = ["pymongo (>=3)"] redis = ["redis (>=3)"] security = ["itsdangerous (>=2.0)"] -yaml = ["pyyaml (>=5.4)"] +yaml = ["pyyaml (>=6.0.1)"] [[package]] name = "requests-mock" @@ -855,19 +1499,19 @@ test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "tes [[package]] name = "setuptools" -version = "69.1.0" +version = "69.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, - {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, + {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, + {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -881,25 +1525,39 @@ files = [ ] [[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" +name = "types-requests" +version = "2.31.0.20240311" +description = "Typing stubs for requests" optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = ">=3.8" files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, + {file = "types-requests-2.31.0.20240311.tar.gz", hash = "sha256:b1c1b66abfb7fa79aae09097a811c4aa97130eb8831c60e47aee4ca344731ca5"}, + {file = "types_requests-2.31.0.20240311-py3-none-any.whl", hash = "sha256:47872893d65a38e282ee9f277a4ee50d1b28bd592040df7d1fdaffdf3779937d"}, ] +[package.dependencies] +urllib3 = ">=2" + [[package]] name = "typing-extensions" -version = "4.9.0" +version = "4.10.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, - {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, ] [[package]] @@ -918,13 +1576,13 @@ six = "*" [[package]] name = "urllib3" -version = "2.2.0" +version = "2.2.1" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, - {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, ] [package.extras] @@ -1026,7 +1684,227 @@ files = [ {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, ] +[[package]] +name = "xxhash" +version = "3.4.1" +description = "Python binding for xxHash" +optional = false +python-versions = ">=3.7" +files = [ + {file = "xxhash-3.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:91dbfa55346ad3e18e738742236554531a621042e419b70ad8f3c1d9c7a16e7f"}, + {file = "xxhash-3.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:665a65c2a48a72068fcc4d21721510df5f51f1142541c890491afc80451636d2"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb11628470a6004dc71a09fe90c2f459ff03d611376c1debeec2d648f44cb693"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5bef2a7dc7b4f4beb45a1edbba9b9194c60a43a89598a87f1a0226d183764189"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c0f7b2d547d72c7eda7aa817acf8791f0146b12b9eba1d4432c531fb0352228"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00f2fdef6b41c9db3d2fc0e7f94cb3db86693e5c45d6de09625caad9a469635b"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23cfd9ca09acaf07a43e5a695143d9a21bf00f5b49b15c07d5388cadf1f9ce11"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6a9ff50a3cf88355ca4731682c168049af1ca222d1d2925ef7119c1a78e95b3b"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f1d7c69a1e9ca5faa75546fdd267f214f63f52f12692f9b3a2f6467c9e67d5e7"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:672b273040d5d5a6864a36287f3514efcd1d4b1b6a7480f294c4b1d1ee1b8de0"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4178f78d70e88f1c4a89ff1ffe9f43147185930bb962ee3979dba15f2b1cc799"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9804b9eb254d4b8cc83ab5a2002128f7d631dd427aa873c8727dba7f1f0d1c2b"}, + {file = "xxhash-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c09c49473212d9c87261d22c74370457cfff5db2ddfc7fd1e35c80c31a8c14ce"}, + {file = "xxhash-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ebbb1616435b4a194ce3466d7247df23499475c7ed4eb2681a1fa42ff766aff6"}, + {file = "xxhash-3.4.1-cp310-cp310-win_arm64.whl", hash = "sha256:25dc66be3db54f8a2d136f695b00cfe88018e59ccff0f3b8f545869f376a8a46"}, + {file = "xxhash-3.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:58c49083801885273e262c0f5bbeac23e520564b8357fbb18fb94ff09d3d3ea5"}, + {file = "xxhash-3.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b526015a973bfbe81e804a586b703f163861da36d186627e27524f5427b0d520"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36ad4457644c91a966f6fe137d7467636bdc51a6ce10a1d04f365c70d6a16d7e"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:248d3e83d119770f96003271fe41e049dd4ae52da2feb8f832b7a20e791d2920"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2070b6d5bbef5ee031666cf21d4953c16e92c2f8a24a94b5c240f8995ba3b1d0"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2746035f518f0410915e247877f7df43ef3372bf36cfa52cc4bc33e85242641"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a8ba6181514681c2591840d5632fcf7356ab287d4aff1c8dea20f3c78097088"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0aac5010869240e95f740de43cd6a05eae180c59edd182ad93bf12ee289484fa"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4cb11d8debab1626181633d184b2372aaa09825bde709bf927704ed72765bed1"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b29728cff2c12f3d9f1d940528ee83918d803c0567866e062683f300d1d2eff3"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a15cbf3a9c40672523bdb6ea97ff74b443406ba0ab9bca10ceccd9546414bd84"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6e66df260fed01ed8ea790c2913271641c58481e807790d9fca8bfd5a3c13844"}, + {file = "xxhash-3.4.1-cp311-cp311-win32.whl", hash = "sha256:e867f68a8f381ea12858e6d67378c05359d3a53a888913b5f7d35fbf68939d5f"}, + {file = "xxhash-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:200a5a3ad9c7c0c02ed1484a1d838b63edcf92ff538770ea07456a3732c577f4"}, + {file = "xxhash-3.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:1d03f1c0d16d24ea032e99f61c552cb2b77d502e545187338bea461fde253583"}, + {file = "xxhash-3.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c4bbba9b182697a52bc0c9f8ec0ba1acb914b4937cd4a877ad78a3b3eeabefb3"}, + {file = "xxhash-3.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9fd28a9da300e64e434cfc96567a8387d9a96e824a9be1452a1e7248b7763b78"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6066d88c9329ab230e18998daec53d819daeee99d003955c8db6fc4971b45ca3"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93805bc3233ad89abf51772f2ed3355097a5dc74e6080de19706fc447da99cd3"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64da57d5ed586ebb2ecdde1e997fa37c27fe32fe61a656b77fabbc58e6fbff6e"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a97322e9a7440bf3c9805cbaac090358b43f650516486746f7fa482672593df"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbe750d512982ee7d831838a5dee9e9848f3fb440e4734cca3f298228cc957a6"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:fd79d4087727daf4d5b8afe594b37d611ab95dc8e29fe1a7517320794837eb7d"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:743612da4071ff9aa4d055f3f111ae5247342931dedb955268954ef7201a71ff"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:b41edaf05734092f24f48c0958b3c6cbaaa5b7e024880692078c6b1f8247e2fc"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:a90356ead70d715fe64c30cd0969072de1860e56b78adf7c69d954b43e29d9fa"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac56eebb364e44c85e1d9e9cc5f6031d78a34f0092fea7fc80478139369a8b4a"}, + {file = "xxhash-3.4.1-cp312-cp312-win32.whl", hash = "sha256:911035345932a153c427107397c1518f8ce456f93c618dd1c5b54ebb22e73747"}, + {file = "xxhash-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:f31ce76489f8601cc7b8713201ce94b4bd7b7ce90ba3353dccce7e9e1fee71fa"}, + {file = "xxhash-3.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:b5beb1c6a72fdc7584102f42c4d9df232ee018ddf806e8c90906547dfb43b2da"}, + {file = "xxhash-3.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6d42b24d1496deb05dee5a24ed510b16de1d6c866c626c2beb11aebf3be278b9"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b685fab18876b14a8f94813fa2ca80cfb5ab6a85d31d5539b7cd749ce9e3624"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:419ffe34c17ae2df019a4685e8d3934d46b2e0bbe46221ab40b7e04ed9f11137"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e041ce5714f95251a88670c114b748bca3bf80cc72400e9f23e6d0d59cf2681"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc860d887c5cb2f524899fb8338e1bb3d5789f75fac179101920d9afddef284b"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:312eba88ffe0a05e332e3a6f9788b73883752be63f8588a6dc1261a3eaaaf2b2"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:e01226b6b6a1ffe4e6bd6d08cfcb3ca708b16f02eb06dd44f3c6e53285f03e4f"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9f3025a0d5d8cf406a9313cd0d5789c77433ba2004b1c75439b67678e5136537"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:6d3472fd4afef2a567d5f14411d94060099901cd8ce9788b22b8c6f13c606a93"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:43984c0a92f06cac434ad181f329a1445017c33807b7ae4f033878d860a4b0f2"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a55e0506fdb09640a82ec4f44171273eeabf6f371a4ec605633adb2837b5d9d5"}, + {file = "xxhash-3.4.1-cp37-cp37m-win32.whl", hash = "sha256:faec30437919555b039a8bdbaba49c013043e8f76c999670aef146d33e05b3a0"}, + {file = "xxhash-3.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:c9e1b646af61f1fc7083bb7b40536be944f1ac67ef5e360bca2d73430186971a"}, + {file = "xxhash-3.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:961d948b7b1c1b6c08484bbce3d489cdf153e4122c3dfb07c2039621243d8795"}, + {file = "xxhash-3.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:719a378930504ab159f7b8e20fa2aa1896cde050011af838af7e7e3518dd82de"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74fb5cb9406ccd7c4dd917f16630d2e5e8cbbb02fc2fca4e559b2a47a64f4940"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5dab508ac39e0ab988039bc7f962c6ad021acd81fd29145962b068df4148c476"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c59f3e46e7daf4c589e8e853d700ef6607afa037bfad32c390175da28127e8c"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cc07256eff0795e0f642df74ad096f8c5d23fe66bc138b83970b50fc7f7f6c5"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9f749999ed80f3955a4af0eb18bb43993f04939350b07b8dd2f44edc98ffee9"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7688d7c02149a90a3d46d55b341ab7ad1b4a3f767be2357e211b4e893efbaaf6"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a8b4977963926f60b0d4f830941c864bed16aa151206c01ad5c531636da5708e"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:8106d88da330f6535a58a8195aa463ef5281a9aa23b04af1848ff715c4398fb4"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4c76a77dbd169450b61c06fd2d5d436189fc8ab7c1571d39265d4822da16df22"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:11f11357c86d83e53719c592021fd524efa9cf024dc7cb1dfb57bbbd0d8713f2"}, + {file = "xxhash-3.4.1-cp38-cp38-win32.whl", hash = "sha256:0c786a6cd74e8765c6809892a0d45886e7c3dc54de4985b4a5eb8b630f3b8e3b"}, + {file = "xxhash-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:aabf37fb8fa27430d50507deeab2ee7b1bcce89910dd10657c38e71fee835594"}, + {file = "xxhash-3.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6127813abc1477f3a83529b6bbcfeddc23162cece76fa69aee8f6a8a97720562"}, + {file = "xxhash-3.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef2e194262f5db16075caea7b3f7f49392242c688412f386d3c7b07c7733a70a"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71be94265b6c6590f0018bbf73759d21a41c6bda20409782d8117e76cd0dfa8b"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10e0a619cdd1c0980e25eb04e30fe96cf8f4324758fa497080af9c21a6de573f"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa122124d2e3bd36581dd78c0efa5f429f5220313479fb1072858188bc2d5ff1"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17032f5a4fea0a074717fe33477cb5ee723a5f428de7563e75af64bfc1b1e10"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca7783b20e3e4f3f52f093538895863f21d18598f9a48211ad757680c3bd006f"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d77d09a1113899fad5f354a1eb4f0a9afcf58cefff51082c8ad643ff890e30cf"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:21287bcdd299fdc3328cc0fbbdeaa46838a1c05391264e51ddb38a3f5b09611f"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:dfd7a6cc483e20b4ad90224aeb589e64ec0f31e5610ab9957ff4314270b2bf31"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:543c7fcbc02bbb4840ea9915134e14dc3dc15cbd5a30873a7a5bf66039db97ec"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fe0a98d990e433013f41827b62be9ab43e3cf18e08b1483fcc343bda0d691182"}, + {file = "xxhash-3.4.1-cp39-cp39-win32.whl", hash = "sha256:b9097af00ebf429cc7c0e7d2fdf28384e4e2e91008130ccda8d5ae653db71e54"}, + {file = "xxhash-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:d699b921af0dcde50ab18be76c0d832f803034d80470703700cb7df0fbec2832"}, + {file = "xxhash-3.4.1-cp39-cp39-win_arm64.whl", hash = "sha256:2be491723405e15cc099ade1280133ccfbf6322d2ef568494fb7d07d280e7eee"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:431625fad7ab5649368c4849d2b49a83dc711b1f20e1f7f04955aab86cd307bc"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc6dbd5fc3c9886a9e041848508b7fb65fd82f94cc793253990f81617b61fe49"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3ff8dbd0ec97aec842476cb8ccc3e17dd288cd6ce3c8ef38bff83d6eb927817"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef73a53fe90558a4096e3256752268a8bdc0322f4692ed928b6cd7ce06ad4fe3"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:450401f42bbd274b519d3d8dcf3c57166913381a3d2664d6609004685039f9d3"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a162840cf4de8a7cd8720ff3b4417fbc10001eefdd2d21541a8226bb5556e3bb"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b736a2a2728ba45017cb67785e03125a79d246462dfa892d023b827007412c52"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d0ae4c2e7698adef58710d6e7a32ff518b66b98854b1c68e70eee504ad061d8"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6322c4291c3ff174dcd104fae41500e75dad12be6f3085d119c2c8a80956c51"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:dd59ed668801c3fae282f8f4edadf6dc7784db6d18139b584b6d9677ddde1b6b"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92693c487e39523a80474b0394645b393f0ae781d8db3474ccdcead0559ccf45"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4603a0f642a1e8d7f3ba5c4c25509aca6a9c1cc16f85091004a7028607ead663"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa45e8cbfbadb40a920fe9ca40c34b393e0b067082d94006f7f64e70c7490a6"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:595b252943b3552de491ff51e5bb79660f84f033977f88f6ca1605846637b7c6"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:562d8b8f783c6af969806aaacf95b6c7b776929ae26c0cd941d54644ea7ef51e"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:41ddeae47cf2828335d8d991f2d2b03b0bdc89289dc64349d712ff8ce59d0647"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c44d584afdf3c4dbb3277e32321d1a7b01d6071c1992524b6543025fb8f4206f"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd7bddb3a5b86213cc3f2c61500c16945a1b80ecd572f3078ddbbe68f9dabdfb"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ecb6c987b62437c2f99c01e97caf8d25660bf541fe79a481d05732e5236719c"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:696b4e18b7023527d5c50ed0626ac0520edac45a50ec7cf3fc265cd08b1f4c03"}, + {file = "xxhash-3.4.1.tar.gz", hash = "sha256:0379d6cf1ff987cd421609a264ce025e74f346e3e145dd106c0cc2e3ec3f99a9"}, +] + +[[package]] +name = "yarl" +version = "1.9.4" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + [metadata] lock-version = "2.0" -python-versions = "^3.9,<3.12" -content-hash = "fde07907def42fe31b6513c3d374b49cd501667cb0368ed468012b58391fb29f" +python-versions = "^3.11" +content-hash = "8d2cfa4b8eb658981f05f0c7419f2df6366fa2edce74439f285147300d919b27" diff --git a/source-notion/pyproject.toml b/source-notion/pyproject.toml index 3c42507630..1d48e9593f 100644 --- a/source-notion/pyproject.toml +++ b/source-notion/pyproject.toml @@ -1,30 +1,24 @@ -[build-system] -requires = [ "poetry-core>=1.0.0",] -build-backend = "poetry.core.masonry.api" - [tool.poetry] -version = "2.1.0" -name = "source-notion" -description = "Source implementation for Notion." -authors = [ "Airbyte ",] -license = "MIT" -readme = "README.md" -documentation = "https://docs.airbyte.com/integrations/sources/notion" -homepage = "https://airbyte.com" -repository = "https://github.com/airbytehq/airbyte" -[[tool.poetry.packages]] -include = "source_notion" +name = "source_notion" +version = "0.1.0" +description = "" +authors = ["Will Baker "] [tool.poetry.dependencies] -python = "^3.9,<3.12" -airbyte-cdk = "==0.52.7" -pendulum = "==2.1.2" - -[tool.poetry.scripts] -source-notion = "source_notion.run:run" +airbyte-cdk = "^0.52" +estuary-cdk = {path="../estuary-cdk", develop = true} +python = "^3.11" +types-requests = "^2.31" [tool.poetry.group.dev.dependencies] +debugpy = "^1.8.0" +mypy = "^1.8.0" +pytest = "^7.4.3" +pytest-insta = "^0.3.0" requests-mock = "^1.11.0" -pytest-mock = "^3.6.1" +pytest-mock = "^3.12.0" freezegun = "^1.4.0" -pytest = "^6.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/source-notion/unit_tests/test_incremental_streams.py b/source-notion/tests/test_incremental_streams.py similarity index 100% rename from source-notion/unit_tests/test_incremental_streams.py rename to source-notion/tests/test_incremental_streams.py diff --git a/source-notion/unit_tests/test_source.py b/source-notion/tests/test_source.py similarity index 100% rename from source-notion/unit_tests/test_source.py rename to source-notion/tests/test_source.py diff --git a/source-notion/unit_tests/test_streams.py b/source-notion/tests/test_streams.py similarity index 100% rename from source-notion/unit_tests/test_streams.py rename to source-notion/tests/test_streams.py diff --git a/source-notion/unit_tests/__init__.py b/source-notion/unit_tests/__init__.py deleted file mode 100644 index 46b7376756..0000000000 --- a/source-notion/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# From 8cce949fa574f3fea0aa209ac9967d538b4ff822 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Mon, 25 Mar 2024 11:38:29 -0400 Subject: [PATCH 53/96] source-notion: add to CI --- .github/workflows/python.yaml | 6 ++++++ source-notion/VERSION | 1 + 2 files changed, 7 insertions(+) create mode 100644 source-notion/VERSION diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 11d84fc9df..6b2261be90 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -13,6 +13,7 @@ on: - "source-google-sheets-native/**" - "source-hubspot-native/**" - "source-hubspot/**" + - "source-notion/**" pull_request: branches: [main] paths: @@ -25,6 +26,7 @@ on: - "source-google-sheets-native/**" - "source-hubspot-native/**" - "source-hubspot/**" + - "source-notion/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -72,6 +74,10 @@ jobs: type: capture version: v5 usage_rate: "1.0" + - name: source-notion + type: capture + version: v2 + usage_rate: "1.0" steps: - uses: actions/checkout@v4 diff --git a/source-notion/VERSION b/source-notion/VERSION new file mode 100644 index 0000000000..8c1384d825 --- /dev/null +++ b/source-notion/VERSION @@ -0,0 +1 @@ +v2 From 2a2236e8817268517c78e7969152334da45aea38 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Thu, 21 Mar 2024 13:27:33 -0500 Subject: [PATCH 54/96] source-sqlserver: Remove `format: time` from `TIME` column type Previously our discovery schema generation for a `TIME` column in the source database produces `{type: string, format: time}`, but that format means "a string satisfying the `full-time` production rule from RFC3339" and `full-time` has to end in a timezone offset suffix, but the `TIME` column is just a bare `HH:MM:SS[.NNN]` value without any associated time zone offset for us to provide, so right now we don't have any such suffix and that means that the values we capture don't actually satisfy the JSON schema we claimed. In order to resolve this we could: 1. Slap on a suffix of `"Z"` to every time value 2. Interpret the time values in the capture's configured timezone. 3. Stop claiming these strings match `format: time` The problem with option 1 is that these times are no generally in the UTC timezone in the first place. The problem with option 2 is that the user might configure a timezone by providing an IANA name like `'America/Chicago'` but that doesn't uniquely identify a fixed UTC offset for a bare `HH:MM:SS` time (basically: should a particular time value be given the DST or non-DST time offset?) and there is no way of resolving that ambiguity which wouldn't lead to unpleasant surprises in at least some cases. So that leaves us with option 3, just outputting the obvious strings representing these time values and removing the format qualifier that we can't actually satisfy. Unlike `DATETIME` columns (where it's worth the effort to satisfy the RFC3339 timestamp requirements because many materializations can use an appropriate destination-specific type on the other side (and also there we have a `YYYY-MM-DD` part which can be used unambiguously with a named timezone)), I do not believe we have much special handling for `format: time` so this shouldn't be any real loss in practice. --- source-sqlserver/datatypes_test.go | 2 +- source-sqlserver/discovery.go | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/source-sqlserver/datatypes_test.go b/source-sqlserver/datatypes_test.go index 4fb098d3a9..2bc49e5768 100644 --- a/source-sqlserver/datatypes_test.go +++ b/source-sqlserver/datatypes_test.go @@ -45,7 +45,7 @@ func TestDatatypes(t *testing.T) { {ColumnType: `image`, ExpectType: `{"type":["string","null"],"contentEncoding":"base64"}`, InputValue: []byte{0x12, 0x34, 0x56, 0x78}, ExpectValue: `"EjRWeA=="`}, {ColumnType: `date`, ExpectType: `{"type":["string","null"],"format":"date"}`, InputValue: `1991-08-31`, ExpectValue: `"1991-08-31"`}, - {ColumnType: `time`, ExpectType: `{"type":["string","null"],"format":"time"}`, InputValue: `12:34:54.125`, ExpectValue: `"12:34:54.125"`}, + {ColumnType: `time`, ExpectType: `{"type":["string","null"]}`, InputValue: `12:34:54.125`, ExpectValue: `"12:34:54.125"`}, {ColumnType: `datetimeoffset`, ExpectType: `{"type":["string","null"],"format":"date-time"}`, InputValue: `1991-08-31T12:34:54.125-06:00`, ExpectValue: `"1991-08-31T12:34:54.125-06:00"`}, {ColumnType: `datetime`, ExpectType: `{"type":["string","null"],"format":"date-time"}`, InputValue: `1991-08-31T12:34:56.789`, ExpectValue: `"1991-08-31T12:34:56.79-05:00"`}, diff --git a/source-sqlserver/discovery.go b/source-sqlserver/discovery.go index 3618e6f926..37e330ca33 100644 --- a/source-sqlserver/discovery.go +++ b/source-sqlserver/discovery.go @@ -386,9 +386,18 @@ var sqlserverTypeToJSON = map[string]columnSchema{ "image": {jsonType: "string", contentEncoding: "base64"}, "date": {jsonType: "string", format: "date"}, - "time": {jsonType: "string", format: "time"}, "datetimeoffset": {jsonType: "string", format: "date-time"}, + // The 'time' format in JSON schemas means the RFC3339 'full-time' grammar rule, + // which includes a numeric timezone offset. The TIME column in SQL Server has + // no associated timezone data, and it's not possible to unambiguously assign a + // numeric timezone offset to these HH:MM:SS time values using the configured + // datetime location (handwaving at one reason: how do we know if DST applies?). + // + // So we don't do that, and that's why TIME columns just get turned into strings + // without a specific format guarantee here. + "time": {jsonType: "string"}, + "uniqueidentifier": {jsonType: "string", format: "uuid"}, "xml": {jsonType: "string"}, From fb60ae6f57f2cf5c0af6b6bc03e1b88300fbc9e2 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Thu, 21 Mar 2024 13:36:21 -0500 Subject: [PATCH 55/96] source-sqlserver: Fix stale test snapshots A couple of SQL Server discovery test snapshots are stale, lacking the changes from https://github.com/estuary/connectors/pull/1371 This commit just updates them to be correct again --- .../TestDiscoveryIrrelevantConstraints | 25 ++++++++++++++++--- .../.snapshots/TestGeneric-KeylessDiscovery | 25 ++++++++++++++++--- .../.snapshots/TestGeneric-SimpleDiscovery | 25 ++++++++++++++++--- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/source-sqlserver/.snapshots/TestDiscoveryIrrelevantConstraints b/source-sqlserver/.snapshots/TestDiscoveryIrrelevantConstraints index 5998597d10..b129cc60a3 100644 --- a/source-sqlserver/.snapshots/TestDiscoveryIrrelevantConstraints +++ b/source-sqlserver/.snapshots/TestDiscoveryIrrelevantConstraints @@ -34,6 +34,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -106,9 +128,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-sqlserver/.snapshots/TestGeneric-KeylessDiscovery b/source-sqlserver/.snapshots/TestGeneric-KeylessDiscovery index 0ed8c5f16a..0f671e8c5b 100644 --- a/source-sqlserver/.snapshots/TestGeneric-KeylessDiscovery +++ b/source-sqlserver/.snapshots/TestGeneric-KeylessDiscovery @@ -38,6 +38,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -110,9 +132,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { diff --git a/source-sqlserver/.snapshots/TestGeneric-SimpleDiscovery b/source-sqlserver/.snapshots/TestGeneric-SimpleDiscovery index 7a0e6fd257..91b4243d05 100644 --- a/source-sqlserver/.snapshots/TestGeneric-SimpleDiscovery +++ b/source-sqlserver/.snapshots/TestGeneric-SimpleDiscovery @@ -37,6 +37,28 @@ Binding 0: }, "allOf": [ { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, "required": [ "_meta" ], @@ -109,9 +131,6 @@ Binding 0: "strategy": "merge" } } - }, - "reduce": { - "strategy": "merge" } }, { From 7aaf2dc6e3f836f16b97dea0a71709f6013a7ff4 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Mon, 25 Mar 2024 13:09:28 -0500 Subject: [PATCH 56/96] source-mysql: Upgrade query event log levels This fixes https://github.com/estuary/connectors/issues/1396 We've observed a couple of cases in production where tables apparently gained columns without our DDL tracking logic picking up on the alteration. Unfortunately we don't have anything in the logs to suggest how that happened, so we pretty much just have to shrug and re-backfill the table and hope it doesn't happen often. So this commit adjusts the log levels so that all 'Query Event' replication messages will appear in the logs at `INFO` level. This could in theory be a little bit spammy, but in practice it should be fine because DML insert/update/delete query events are a fatal error (since `binlog_format=ROW` should mean that those changes always come in as row alteration events) and one log line per DDL query shouldn't be too bad in real-world usage. --- source-mysql/replication.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source-mysql/replication.go b/source-mysql/replication.go index 46d22ba62b..2d4a6c8619 100644 --- a/source-mysql/replication.go +++ b/source-mysql/replication.go @@ -468,10 +468,10 @@ func (rs *mysqlReplicationStream) handleQuery(ctx context.Context, schema, query // don't impact our capture or because we get the relevant information // by some other means. if ignoreQueriesRe.MatchString(query) { - logrus.WithField("query", query).Trace("ignoring query event") + logrus.WithField("query", query).Info("ignoring query event") return nil } - logrus.WithField("query", query).Debug("handling query event") + logrus.WithField("query", query).Info("handling query event") var stmt, err = sqlparser.Parse(query) if err != nil { From 30b6b75decf191c1a0286b93a561ccbcb974c0f1 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Mon, 25 Mar 2024 13:07:16 -0500 Subject: [PATCH 57/96] source-mysql: Ignore `ALTER DATABASE` query events None of the database-level alterations are things we need to care about. --- source-mysql/replication.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source-mysql/replication.go b/source-mysql/replication.go index 2d4a6c8619..e5aaf33721 100644 --- a/source-mysql/replication.go +++ b/source-mysql/replication.go @@ -480,7 +480,7 @@ func (rs *mysqlReplicationStream) handleQuery(ctx context.Context, schema, query logrus.WithField("stmt", fmt.Sprintf("%#v", stmt)).Debug("parsed query") switch stmt := stmt.(type) { - case *sqlparser.CreateDatabase, *sqlparser.CreateTable, *sqlparser.Savepoint, *sqlparser.Flush: + case *sqlparser.CreateDatabase, *sqlparser.AlterDatabase, *sqlparser.CreateTable, *sqlparser.Savepoint, *sqlparser.Flush: logrus.WithField("query", query).Debug("ignoring benign query") case *sqlparser.CreateView, *sqlparser.AlterView, *sqlparser.DropView: // All view creation/deletion/alterations should be fine to ignore since we don't capture from views. From 95c6a54475389f29727216b3fac4c279fe578a83 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Mon, 25 Mar 2024 14:08:19 -0500 Subject: [PATCH 58/96] source-mysql: Silently ignore a couple of query events after all Turns out that in https://github.com/estuary/connectors/pull/1412 I was overly optimistic about the rarity of certain non-DDL query events in the binlogs. The two most egregiously spammy forms are: "BEGIN" "# Dummy event replacing event type 160 that slave cannot handle. " But I'm still skeptical about the `ignoreQueriesRe` in general and suspect that one or more of those query event regexps is responsible for us missing out on DDL operations we care about. So I'm introducing a quick-and-dirty third tier here, a regexp for only the most spammy of query events which uses more conservative patterns so we're *really* sure anything matching that can be ignored without even logging it (at `INFO` level). --- source-mysql/replication.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source-mysql/replication.go b/source-mysql/replication.go index e5aaf33721..17bdab7494 100644 --- a/source-mysql/replication.go +++ b/source-mysql/replication.go @@ -452,6 +452,7 @@ func decodeRow(streamID string, colNames []string, row []interface{}) (map[strin // with the binlog Query Events for some statements like GRANT and CREATE USER. // TODO(johnny): SET STATEMENT is not safe in the general case, and we want to re-visit // by extracting and ignoring a SET STATEMENT stanza prior to parsing. +var silentIgnoreQueriesRe = regexp.MustCompile(`(?i)^(BEGIN|# [^\n]*)$`) var ignoreQueriesRe = regexp.MustCompile(`(?i)^(BEGIN|COMMIT|GRANT|REVOKE|CREATE USER|CREATE DEFINER|DROP USER|ALTER USER|DROP PROCEDURE|DROP FUNCTION|DROP TRIGGER|SET STATEMENT|# |/\*|-- )`) func (rs *mysqlReplicationStream) handleQuery(ctx context.Context, schema, query string) error { @@ -467,6 +468,10 @@ func (rs *mysqlReplicationStream) handleQuery(ctx context.Context, schema, query // that we don't care about, either because they change things that // don't impact our capture or because we get the relevant information // by some other means. + if silentIgnoreQueriesRe.MatchString(query) { + logrus.WithField("query", query).Trace("silently ignoring query event") + return nil + } if ignoreQueriesRe.MatchString(query) { logrus.WithField("query", query).Info("ignoring query event") return nil From 4528ff68d2b3775f876f5f10be891a856ae35025 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Mon, 25 Mar 2024 13:59:00 -0500 Subject: [PATCH 59/96] source-mysql: Check for `--binlog-do-db` not including `"flow"` This fixes https://github.com/estuary/connectors/issues/1340 --- source-mysql/prerequisites.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/source-mysql/prerequisites.go b/source-mysql/prerequisites.go index 2cfb456ea6..0a4fed792e 100644 --- a/source-mysql/prerequisites.go +++ b/source-mysql/prerequisites.go @@ -192,12 +192,44 @@ func (db *mysqlDatabase) prerequisiteUserPermissions(ctx context.Context) error if err != nil { return fmt.Errorf("user %q needs the REPLICATION CLIENT permission", db.config.User) } + if len(results.Values) == 0 { // This failure condition has nothing to do with user permissions, but since we're // already running SHOW MASTER STATUS it would be redundant to do it again in a separate // check just to verify that the result is non-empty. return fmt.Errorf("unable to query latest binlog position (is binary logging enabled?)") } + + // The result of a `SHOW MASTER STATUS` query also tells us if only specific schemas are + // written to the binlog because the user specified --binlog-do-db startup flags. There are + // a lot of ways this could theoretically be misconfigured so we won't bother trying to check + // for every possible misconfiguration, but the specific case of "the schemas of interest are + // logged but the schema containing the Flow watermarks table isn't" occurs often enough that + // it's worth explicitly checking and producing a nice error in that case. + for _, rowValues := range results.Values { + // Translate the result row into a map so that minor changes we don't care + // about don't impact our processing. + var row = make(map[string]any) + for colIdx, colValue := range rowValues { + var key = string(results.Fields[colIdx].Name) + var val = colValue.Value() + if bs, ok := val.([]byte); ok { + val = string(bs) + } + row[key] = val + } + + // Blindly splitting and indexing is valid here because validation for the Config + // struct already checked that the watermarks table name is fully-qualified. + var watermarksSchema = strings.Split(db.config.Advanced.WatermarksTable, ".")[0] + + // The use of strings.Contains here could result in false negatives, but this is + // much less of a concern than potential false positives and the extra code needed + // to parse this column value into an actual list of schema names. + if doDB, ok := row["Binlog_Do_DB"].(string); ok && doDB != "" && !strings.Contains(doDB, watermarksSchema) { + return fmt.Errorf("binlog-do-db is set to %q, which doesn't include the watermark table schema %q", doDB, watermarksSchema) + } + } results.Close() // The SHOW SLAVE HOSTS command (called SHOW REPLICAS in newer versions, but we're From 2b03a286e6bcb5a5399ecd9f2033efae9f446612 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Tue, 26 Mar 2024 13:29:24 -0500 Subject: [PATCH 60/96] source-sqlserver: Disable DB version prerequisite check Currently this check is *way* too conservative, because we know that the SQL Server CDC functionality has existed in its current form since at least SQL Server 2008 (though not all versions of the database). In the absence of this check, a database which doesn't support CDC will instead produce several errors, something like: - CDC is not enabled on database "db" and user "flow_capture" cannot enable it - unable to query capture instances for table "dbo.flow_watermarks": - error querying the current LSN: (In case it wasn't clear I haven't actually run this against such a database just now, so I'm handwaving exactly what the database- provided portion of those errors will look like). This seems like a reasonable tradeoff since right now our stubborn insistence on a minimum database version blocks real users with old databases from setting up a capture, and after all the work we went to around text collations I am not aware of any other issues with capturing from such DBs. --- source-sqlserver/prerequisites.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source-sqlserver/prerequisites.go b/source-sqlserver/prerequisites.go index 3e84add416..12d25ccbaf 100644 --- a/source-sqlserver/prerequisites.go +++ b/source-sqlserver/prerequisites.go @@ -13,12 +13,12 @@ import ( func (db *sqlserverDatabase) SetupPrerequisites(ctx context.Context) []error { var errs []error - if err := db.prerequisiteVersion(ctx); err != nil { - // Return early if the database version is incompatible with the connector since additional - // errors will be of minimal use. - errs = append(errs, err) - return errs - } + //if err := db.prerequisiteVersion(ctx); err != nil { + // // Return early if the database version is incompatible with the connector since additional + // // errors will be of minimal use. + // errs = append(errs, err) + // return errs + //} for _, prereq := range []func(ctx context.Context) error{ db.prerequisiteCDCEnabled, From baca9d5eb3049ec093c157909e4db5de78677e6e Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 27 Mar 2024 14:17:50 +0000 Subject: [PATCH 61/96] stdsql: close connection used to ping database in exec --- materialize-sql/std_sql.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/materialize-sql/std_sql.go b/materialize-sql/std_sql.go index 2f2bb05d76..289c98c8fb 100644 --- a/materialize-sql/std_sql.go +++ b/materialize-sql/std_sql.go @@ -46,12 +46,14 @@ func StdSQLExecStatements(ctx context.Context, db *sql.DB, statements []string) // through a single connection. This allows a driver to explicitly run // `BEGIN;` and `COMMIT;` statements around a transactional operation. var conn, err = db.Conn(ctx) - if err == nil { - err = conn.PingContext(ctx) - } if err != nil { return fmt.Errorf("connecting to DB: %w", err) } + defer conn.Close() + + if err = conn.PingContext(ctx); err != nil { + return fmt.Errorf("ping DB: %w", err) + } for _, statement := range statements { if _, err := conn.ExecContext(ctx, statement); err != nil { @@ -59,7 +61,8 @@ func StdSQLExecStatements(ctx context.Context, db *sql.DB, statements []string) } logrus.WithField("sql", statement).Debug("executed statement") } - return conn.Close() // Release to pool. + + return nil } // StdInstallFence is a convenience for Client implementations which From 4a381e512856a3b44a4ee0fc0bed73a7998b31b3 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 27 Mar 2024 17:32:01 +0000 Subject: [PATCH 62/96] materialize-databricks: start warehouse using API before ping --- materialize-databricks/client.go | 42 +++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/materialize-databricks/client.go b/materialize-databricks/client.go index 5176b120ca..fe5654ea36 100644 --- a/materialize-databricks/client.go +++ b/materialize-databricks/client.go @@ -175,6 +175,8 @@ func (c *client) PreReqs(ctx context.Context) *sql.PrereqErr { var httpPathSplit = strings.Split(c.cfg.HTTPPath, "/") var warehouseId = httpPathSplit[len(httpPathSplit)-1] + var warehouseStopped = true + var warehouseErr error if res, err := wsClient.Warehouses.GetById(ctx, warehouseId); err != nil { errs.Err(err) } else { @@ -184,23 +186,41 @@ func (c *client) PreReqs(ctx context.Context) *sql.PrereqErr { case databricksSql.StateDeleting: errs.Err(fmt.Errorf("The selected SQL Warehouse is being deleted, please use an active SQL warehouse.")) case databricksSql.StateStarting: - errs.Err(fmt.Errorf("The selected SQL Warehouse is starting, please wait a couple of minutes before trying again.")) + warehouseErr = fmt.Errorf("The selected SQL Warehouse is starting, please wait a couple of minutes before trying again.") case databricksSql.StateStopped: - errs.Err(fmt.Errorf("The selected SQL Warehouse is stopped, please start the SQL warehouse and try again.")) + warehouseErr = fmt.Errorf("The selected SQL Warehouse is stopped, please start the SQL warehouse and try again.") case databricksSql.StateStopping: - errs.Err(fmt.Errorf("The selected SQL Warehouse is stopping, please start the SQL warehouse and try again.")) + warehouseErr = fmt.Errorf("The selected SQL Warehouse is stopping, please start the SQL warehouse and try again.") + case databricksSql.StateRunning: + warehouseStopped = false } } - // Use a reasonable timeout for this connection test. It is not uncommon for a misconfigured - // connection (wrong host, wrong port, etc.) to hang for several minutes on Ping and we want to - // bail out well before then. Note that it is normal for Databricks warehouses to go offline - // after inactivity, and this attempt to connect to the warehouse will initiate their boot-up - // process however we don't want to wait 5 minutes as that does not create a good UX for the - // user in the UI - ctx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() + if errs.Len() > 0 { + return errs + } + + if warehouseStopped { + // Use a reasonable timeout for this connection test. It is not uncommon for a misconfigured + // connection (wrong host, wrong port, etc.) to hang for several minutes on Ping and we want to + // bail out well before then. Note that it is normal for Databricks warehouses to go offline + // after inactivity, and this attempt to connect to the warehouse will initiate their boot-up + // process however we don't want to wait 5 minutes as that does not create a good UX for the + // user in the UI + if r, err := wsClient.Warehouses.Start(ctx, databricksSql.StartRequest{Id: warehouseId}); err != nil { + errs.Err(fmt.Errorf("Could not start the warehouse: %w", err)) + } else if _, err := r.GetWithTimeout(60 * time.Second); err != nil { + errs.Err(warehouseErr) + } + + if errs.Len() > 0 { + return errs + } + } + // We avoid running this ping if the warehouse is not awake, see + // the issue below for more information on why: + // https://github.com/databricks/databricks-sql-go/issues/198 if err := c.db.PingContext(ctx); err != nil { // Provide a more user-friendly representation of some common error causes. var execErr dbsqlerr.DBExecutionError From 01a92fb8acd65043b5592e083dfa5ae95d26cef9 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 27 Mar 2024 17:59:29 +0000 Subject: [PATCH 63/96] materialize-sql: return conn.Close err from defer in exec --- materialize-sql/std_sql.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/materialize-sql/std_sql.go b/materialize-sql/std_sql.go index 289c98c8fb..3008ea90f4 100644 --- a/materialize-sql/std_sql.go +++ b/materialize-sql/std_sql.go @@ -49,7 +49,9 @@ func StdSQLExecStatements(ctx context.Context, db *sql.DB, statements []string) if err != nil { return fmt.Errorf("connecting to DB: %w", err) } - defer conn.Close() + defer func() { + err = conn.Close() + }() if err = conn.PingContext(ctx); err != nil { return fmt.Errorf("ping DB: %w", err) @@ -62,7 +64,7 @@ func StdSQLExecStatements(ctx context.Context, db *sql.DB, statements []string) logrus.WithField("sql", statement).Debug("executed statement") } - return nil + return err } // StdInstallFence is a convenience for Client implementations which From 2bbd72576df4f0965a826a600f632ad425c53129 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Wed, 27 Mar 2024 12:53:47 -0500 Subject: [PATCH 64/96] source-sqlserver: Query job status during replication diagnostics Since SQL Server replication works by issuing polling queries and most ways that can fail result in errors, a "silent failure" mode seems most likely to come about if the CDC agent job whose job it is to tail the WAL and copy changes into change tables simply isn't running. The new diagnostic logic is untested, but it will only be run in the event that replication is already not working as intended, so the risk of breakage is minimal here and I intend to try it out in production shortly after it's live. --- source-sqlserver/replication.go | 42 ++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/source-sqlserver/replication.go b/source-sqlserver/replication.go index 37dd380af3..cd579dea93 100644 --- a/source-sqlserver/replication.go +++ b/source-sqlserver/replication.go @@ -422,6 +422,46 @@ func splitStreamID(streamID string) (string, string) { } func (db *sqlserverDatabase) ReplicationDiagnostics(ctx context.Context) error { - // TODO: Run some useful diagnostics queries for SQL Server replication and log the results + var query = func(q string) { + var logEntry = log.WithField("query", q) + logEntry.Info("running diagnostics query") + + var rows, err = db.conn.QueryContext(ctx, q) + if err != nil { + logEntry.WithField("err", err).Error("unable to execute diagnostics query") + return + } + defer rows.Close() + + cnames, err := rows.Columns() + if err != nil { + logEntry.WithField("err", err).Error("error processing query result") + return + } + var vals = make([]any, len(cnames)) + var vptrs = make([]any, len(vals)) + for idx := range vals { + vptrs[idx] = &vals[idx] + } + + var numResults int + for rows.Next() { + numResults++ + if err := rows.Scan(vptrs...); err != nil { + logEntry.WithField("err", err).Error("error scanning result row") + continue + } + var logFields = log.Fields{} + for idx, name := range cnames { + logFields[name] = vals[idx] + } + log.WithFields(logFields).Info("got row") + } + if numResults == 0 { + logEntry.Info("no results") + } + } + + query("EXEC msdb.dbo.sp_help_job;") return nil } From ae6556f593cf4eca980887790115c5ca9bbe9152 Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 28 Mar 2024 12:58:32 -0300 Subject: [PATCH 65/96] import: source-linkedin-pages through airbyte @ c613853d10700c60d2edd44dd243a809340d8c1c Remote Repo URL: https://github.com/airbytehq/airbyte.git Source name: c613853d10700c60d2edd44dd243a809340d8c1c Source Commit ID: c613853d10700c60d2edd44dd243a809340d8c1c Source Repo Prefix: airbyte-integrations/connectors/source-linkedin-pages/ Import Path: source-linkedin-pages/ License Type: MIT License Path: ./airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml git-merge-subpath: c613853d10700c60d2edd44dd243a809340d8c1c airbyte-integrations/connectors/source-linkedin-pages source-linkedin-pages --- source-linkedin-pages/.dockerignore | 6 + source-linkedin-pages/Dockerfile | 38 ++ source-linkedin-pages/README.md | 100 ++++++ .../acceptance-test-config.yml | 29 ++ source-linkedin-pages/bootstrap.md | 7 + source-linkedin-pages/icon.svg | 1 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 23 ++ .../integration_tests/acceptance.py | 13 + .../integration_tests/configured_catalog.json | 40 +++ .../integration_tests/invalid_config.json | 7 + .../integration_tests/sample_config.json | 7 + .../integration_tests/sample_state.json | 5 + source-linkedin-pages/main.py | 8 + source-linkedin-pages/metadata.yaml | 29 ++ source-linkedin-pages/requirements.txt | 1 + source-linkedin-pages/setup.py | 47 +++ .../source_linkedin_pages/__init__.py | 8 + .../source_linkedin_pages/run.py | 14 + .../schemas/follower_statistics.json | 59 ++++ .../schemas/organization_lookup.json | 278 +++++++++++++++ .../schemas/share_statistics.json | 40 +++ .../schemas/total_follower_count.json | 10 + .../source_linkedin_pages/source.py | 152 ++++++++ .../source_linkedin_pages/spec.json | 71 ++++ .../source_linkedin_pages/utils.py | 324 ++++++++++++++++++ source-linkedin-pages/unit_tests/__init__.py | 3 + 27 files changed, 1323 insertions(+) create mode 100644 source-linkedin-pages/.dockerignore create mode 100644 source-linkedin-pages/Dockerfile create mode 100644 source-linkedin-pages/README.md create mode 100644 source-linkedin-pages/acceptance-test-config.yml create mode 100644 source-linkedin-pages/bootstrap.md create mode 100644 source-linkedin-pages/icon.svg create mode 100644 source-linkedin-pages/integration_tests/__init__.py create mode 100644 source-linkedin-pages/integration_tests/abnormal_state.json create mode 100644 source-linkedin-pages/integration_tests/acceptance.py create mode 100644 source-linkedin-pages/integration_tests/configured_catalog.json create mode 100644 source-linkedin-pages/integration_tests/invalid_config.json create mode 100644 source-linkedin-pages/integration_tests/sample_config.json create mode 100644 source-linkedin-pages/integration_tests/sample_state.json create mode 100644 source-linkedin-pages/main.py create mode 100644 source-linkedin-pages/metadata.yaml create mode 100644 source-linkedin-pages/requirements.txt create mode 100644 source-linkedin-pages/setup.py create mode 100644 source-linkedin-pages/source_linkedin_pages/__init__.py create mode 100644 source-linkedin-pages/source_linkedin_pages/run.py create mode 100644 source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json create mode 100644 source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json create mode 100644 source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json create mode 100644 source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json create mode 100644 source-linkedin-pages/source_linkedin_pages/source.py create mode 100644 source-linkedin-pages/source_linkedin_pages/spec.json create mode 100644 source-linkedin-pages/source_linkedin_pages/utils.py create mode 100644 source-linkedin-pages/unit_tests/__init__.py diff --git a/source-linkedin-pages/.dockerignore b/source-linkedin-pages/.dockerignore new file mode 100644 index 0000000000..53de6536d1 --- /dev/null +++ b/source-linkedin-pages/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_linkedin_pages +!setup.py +!secrets diff --git a/source-linkedin-pages/Dockerfile b/source-linkedin-pages/Dockerfile new file mode 100644 index 0000000000..674e988746 --- /dev/null +++ b/source-linkedin-pages/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_linkedin_pages ./source_linkedin_pages + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=1.0.2 +LABEL io.airbyte.name=airbyte/source-linkedin-pages diff --git a/source-linkedin-pages/README.md b/source-linkedin-pages/README.md new file mode 100644 index 0000000000..aa8934c3e3 --- /dev/null +++ b/source-linkedin-pages/README.md @@ -0,0 +1,100 @@ +# Linkedin Pages Source + +This is the repository for the Linkedin Pages source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/linkedin-pages). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linkedin-pages) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linkedin_pages/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source linkedin-pages test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + + +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-linkedin-pages build +``` + +An image will be built with the tag `airbyte/source-linkedin-pages:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-linkedin-pages:dev . +``` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-linkedin-pages:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-linkedin-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-linkedin-pages test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-linkedin-pages test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/linkedin-pages.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/source-linkedin-pages/acceptance-test-config.yml b/source-linkedin-pages/acceptance-test-config.yml new file mode 100644 index 0000000000..535ba5a472 --- /dev/null +++ b/source-linkedin-pages/acceptance-test-config.yml @@ -0,0 +1,29 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-linkedin-pages:dev +test_strictness_level: low +acceptance_tests: + spec: + tests: + - spec_path: "source_linkedin_pages/spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/source-linkedin-pages/bootstrap.md b/source-linkedin-pages/bootstrap.md new file mode 100644 index 0000000000..90e5cbd5eb --- /dev/null +++ b/source-linkedin-pages/bootstrap.md @@ -0,0 +1,7 @@ +The LinkedIn Marketing Developer Platform API can be used to pull data from LinkedIn Organizations such as company page details, follower counts, follower statistics, and all sorts of organic content data. + +You must have a LinkedIn Developers' App created in order to request access to the Marketing Developer Platform API. API Access approval takes up to 72 hours after submitting a ~15-question form. + +The app also must be verified by an admin of the LinkedIn organization your app is created for. Once the app is "verified" and granted access to the Marketing Developer Platform API, you can use their easy-peasy OAuth Token Tools to generate access tokens **and** refresh tokens. + +You can access the `client id` and `client secret` in the **Auth** tab of the app dashboard to round out all of the authorization needs you may have. \ No newline at end of file diff --git a/source-linkedin-pages/icon.svg b/source-linkedin-pages/icon.svg new file mode 100644 index 0000000000..246cc94447 --- /dev/null +++ b/source-linkedin-pages/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/source-linkedin-pages/integration_tests/__init__.py b/source-linkedin-pages/integration_tests/__init__.py new file mode 100644 index 0000000000..46b7376756 --- /dev/null +++ b/source-linkedin-pages/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/source-linkedin-pages/integration_tests/abnormal_state.json b/source-linkedin-pages/integration_tests/abnormal_state.json new file mode 100644 index 0000000000..8f9f142bd9 --- /dev/null +++ b/source-linkedin-pages/integration_tests/abnormal_state.json @@ -0,0 +1,23 @@ +{ + "organization_lookup": { + "lastModified": "2050-01-01" + }, + "follower_statistics": { + "lastModified": "2050-01-01" + }, + "page_statistics": { + "lastModified": "2050-01-01" + }, + "share_statistics": { + "lastModified": "2050-01-01" + }, + "shares": { + "lastModified": "2050-01-01" + }, + "total_follower_count": { + "end_date": "2050-01-01" + }, + "ugc_posts": { + "end_date": "2050-01-01" + } +} diff --git a/source-linkedin-pages/integration_tests/acceptance.py b/source-linkedin-pages/integration_tests/acceptance.py new file mode 100644 index 0000000000..d49b558823 --- /dev/null +++ b/source-linkedin-pages/integration_tests/acceptance.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + yield diff --git a/source-linkedin-pages/integration_tests/configured_catalog.json b/source-linkedin-pages/integration_tests/configured_catalog.json new file mode 100644 index 0000000000..c0e203e604 --- /dev/null +++ b/source-linkedin-pages/integration_tests/configured_catalog.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "organization_lookup", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "follower_statistics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "share_statistics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "total_follower_count", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/source-linkedin-pages/integration_tests/invalid_config.json b/source-linkedin-pages/integration_tests/invalid_config.json new file mode 100644 index 0000000000..a0fa475e00 --- /dev/null +++ b/source-linkedin-pages/integration_tests/invalid_config.json @@ -0,0 +1,7 @@ +{ + "org_id": "12345678", + "credentials": { + "auth_method": "access_token", + "access_token": "wrong_token_sra6ibiw0ZWEdMnC0ZizeD1gLRQP6u1pkQl" + } +} diff --git a/source-linkedin-pages/integration_tests/sample_config.json b/source-linkedin-pages/integration_tests/sample_config.json new file mode 100644 index 0000000000..7399ec4e79 --- /dev/null +++ b/source-linkedin-pages/integration_tests/sample_config.json @@ -0,0 +1,7 @@ +{ + "org_id": "12345678", + "credentials": { + "auth_method": "access_token", + "access_token": "example_token_string123" + } +} diff --git a/source-linkedin-pages/integration_tests/sample_state.json b/source-linkedin-pages/integration_tests/sample_state.json new file mode 100644 index 0000000000..3587e57982 --- /dev/null +++ b/source-linkedin-pages/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/source-linkedin-pages/main.py b/source-linkedin-pages/main.py new file mode 100644 index 0000000000..a6068720a6 --- /dev/null +++ b/source-linkedin-pages/main.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_linkedin_pages.run import run + +if __name__ == "__main__": + run() diff --git a/source-linkedin-pages/metadata.yaml b/source-linkedin-pages/metadata.yaml new file mode 100644 index 0000000000..3d2772011d --- /dev/null +++ b/source-linkedin-pages/metadata.yaml @@ -0,0 +1,29 @@ +data: + ab_internal: + ql: 200 + sl: 100 + connectorSubtype: api + connectorType: source + definitionId: af54297c-e8f8-4d63-a00d-a94695acc9d3 + dockerImageTag: 1.0.2 + dockerRepository: airbyte/source-linkedin-pages + documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-pages + githubIssueLabel: source-linkedin-pages + icon: linkedin.svg + license: MIT + name: LinkedIn Pages + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-linkedin-pages + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: alpha + supportLevel: community + tags: + - language:python + - cdk:python +metadataSpecVersion: "1.0" diff --git a/source-linkedin-pages/requirements.txt b/source-linkedin-pages/requirements.txt new file mode 100644 index 0000000000..d6e1198b1a --- /dev/null +++ b/source-linkedin-pages/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/source-linkedin-pages/setup.py b/source-linkedin-pages/setup.py new file mode 100644 index 0000000000..1b491a1f3e --- /dev/null +++ b/source-linkedin-pages/setup.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", + "pendulum~=2.1", +] + +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.1", + "pytest-mock~=3.6.1", +] + +setup( + entry_points={ + "console_scripts": [ + "source-linkedin-pages=source_linkedin_pages.run:run", + ], + }, + name="source_linkedin_pages", + description="Source implementation for Linkedin Company Pages.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={ + "": [ + # Include yaml files in the package (if any) + "*.yml", + "*.yaml", + # Include all json files in the package, up to 4 levels deep + "*.json", + "*/*.json", + "*/*/*.json", + "*/*/*/*.json", + "*/*/*/*/*.json", + ] + }, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/source-linkedin-pages/source_linkedin_pages/__init__.py b/source-linkedin-pages/source_linkedin_pages/__init__.py new file mode 100644 index 0000000000..e4157287bd --- /dev/null +++ b/source-linkedin-pages/source_linkedin_pages/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceLinkedinPages + +__all__ = ["SourceLinkedinPages"] diff --git a/source-linkedin-pages/source_linkedin_pages/run.py b/source-linkedin-pages/source_linkedin_pages/run.py new file mode 100644 index 0000000000..4f0787573f --- /dev/null +++ b/source-linkedin-pages/source_linkedin_pages/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linkedin_pages import SourceLinkedinPages + + +def run(): + source = SourceLinkedinPages() + launch(source, sys.argv[1:]) diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json new file mode 100644 index 0000000000..f29db87853 --- /dev/null +++ b/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -0,0 +1,59 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "followerCountsByStaffCountRange": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, + "followerCountsByFunction": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, + "followerCountsByAssociationType": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, + "followerCountsBySeniority": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, + "followerCountsByRegion": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, + "organizationalEntity": { + "type": ["null", "string"] + }, + "followerCountsByCountry": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, + "followerCountsByIndustry": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + } + } +} diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json new file mode 100644 index 0000000000..34d3275870 --- /dev/null +++ b/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -0,0 +1,278 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "vanityName": { + "type": ["null", "string"] + }, + "localizedName": { + "type": ["null", "string"] + }, + "website": { + "type": ["null", "object"], + "properties": { + "localized": { + "type": ["null", "object"], + "properties": { + "en_US": { + "type": ["null", "string"] + } + } + }, + "preferredLocale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + } + } + }, + "foundedOn": { + "type": ["null", "object"], + "properties": { + "year": { + "type": ["null", "integer"] + } + } + }, + "groups": { + "type": ["null", "array"], + "items": { + "items": {} + } + }, + "description": { + "type": ["null", "object"], + "properties": { + "localized": { + "type": ["null", "object"], + "properties": { + "en_US": { + "type": ["null", "string"] + } + } + }, + "preferredLocale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + } + } + }, + "versionTag": { + "type": ["null", "string"] + }, + "coverPhotoV2": { + "type": ["null", "object"], + "properties": { + "cropped": { + "type": ["null", "string"] + }, + "original": { + "type": ["null", "string"] + }, + "cropInfo": { + "type": ["null", "object"], + "properties": { + "x": { + "type": ["null", "integer"] + }, + "width": { + "type": ["null", "integer"] + }, + "y": { + "type": ["null", "integer"] + }, + "height": { + "type": ["null", "integer"] + } + } + } + } + }, + "defaultLocale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + }, + "organizationType": { + "type": ["null", "string"] + }, + "alternativeNames": { + "type": ["null", "array"], + "items": { + "items": {} + } + }, + "specialties": { + "type": ["null", "array"], + "items": { + "items": {} + } + }, + "staffCountRange": { + "type": ["null", "string"] + }, + "localizedSpecialties": { + "type": ["null", "array"], + "items": { + "items": {} + } + }, + "industries": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "name": { + "type": ["null", "object"], + "properties": { + "localized": { + "type": ["null", "object"], + "properties": { + "en_US": { + "type": ["null", "string"] + } + } + }, + "preferredLocale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + } + } + }, + "primaryOrganizationType": { + "type": ["null", "string"] + }, + "locations": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "locationType": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "object"], + "properties": { + "localized": { + "type": ["null", "object"], + "properties": { + "en_US": { + "type": ["null", "string"] + } + } + }, + "preferredLocale": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + } + } + } + } + }, + "address": { + "type": ["null", "object"], + "properties": { + "geographicArea": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + } + } + }, + "localizedDescription": { + "type": ["null", "string"] + }, + "geoLocation": { + "type": ["null", "string"] + }, + "streetAddressFieldState": { + "type": ["null", "string"] + } + } + } + }, + "id": { + "type": ["null", "integer"] + }, + "localizedDescription": { + "type": ["null", "string"] + }, + "$URN": { + "type": ["null", "string"] + }, + "localizedWebsite": { + "type": ["null", "string"] + }, + "logoV2": { + "type": ["null", "object"], + "properties": { + "cropped": { + "type": ["null", "string"] + }, + "original": { + "type": ["null", "string"] + }, + "cropInfo": { + "type": ["null", "object"], + "properties": { + "x": { + "type": ["null", "integer"] + }, + "width": { + "type": ["null", "integer"] + }, + "y": { + "type": ["null", "integer"] + }, + "height": { + "type": ["null", "integer"] + } + } + } + } + } + } +} diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json new file mode 100644 index 0000000000..77129bc9d5 --- /dev/null +++ b/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "totalShareStatistics": { + "type": ["null", "object"], + "properties": { + "uniqueImpressionsCount": { + "type": ["null", "integer"] + }, + "clickCount": { + "type": ["null", "integer"] + }, + "engagement": { + "type": ["null", "number"] + }, + "likeCount": { + "type": ["null", "integer"] + }, + "commentCount": { + "type": ["null", "integer"] + }, + "shareCount": { + "type": ["null", "integer"] + }, + "commentMentionsCount": { + "type": ["null", "integer"] + }, + "impressionCount": { + "type": ["null", "integer"] + }, + "shareMentionsCount": { + "type": ["null", "integer"] + } + } + }, + "organizationalEntity": { "type": ["null", "string"] } + } +} diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json b/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json new file mode 100644 index 0000000000..84387c6c37 --- /dev/null +++ b/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "firstDegreeSize": { + "type": ["null", "integer"] + } + } +} diff --git a/source-linkedin-pages/source_linkedin_pages/source.py b/source-linkedin-pages/source_linkedin_pages/source.py new file mode 100644 index 0000000000..93b7ef3f75 --- /dev/null +++ b/source-linkedin-pages/source_linkedin_pages/source.py @@ -0,0 +1,152 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator + + +class LinkedinPagesStream(HttpStream, ABC): + + url_base = "https://api.linkedin.com/v2/" + primary_key = None + + def __init__(self, config): + super().__init__(authenticator=config.get("authenticator")) + self.config = config + + @property + def org(self): + """Property to return the user Organization Id from input""" + return self.config.get("org_id") + + def path(self, **kwargs) -> str: + """Returns the API endpoint path for stream, from `endpoint` class attribute.""" + return self.endpoint + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + ) -> Iterable[Mapping]: + return [response.json()] + + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == 429: + error_message = ( + f"Stream {self.name}: LinkedIn API requests are rate limited. " + f"Rate limits specify the maximum number of API calls that can be made in a 24 hour period. " + f"These limits reset at midnight UTC every day. " + f"You can find more information here https://docs.airbyte.com/integrations/sources/linkedin-pages. " + f"Also quotas and usage are here: https://www.linkedin.com/developers/apps." + ) + self.logger.error(error_message) + return super().should_retry(response) + + +class OrganizationLookup(LinkedinPagesStream): + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizations/{self.org}" + return path + + +class FollowerStatistics(LinkedinPagesStream): + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizationalEntityFollowerStatistics?q=organizationalEntity&organizationalEntity=urn:li:organization:{self.org}" + return path + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + ) -> Iterable[Mapping]: + yield from response.json().get("elements") + + +class ShareStatistics(LinkedinPagesStream): + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3A{self.org}" + return path + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + ) -> Iterable[Mapping]: + yield from response.json().get("elements") + + +class TotalFollowerCount(LinkedinPagesStream): + def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + + path = f"networkSizes/urn:li:organization:{self.org}?edgeType=CompanyFollowedByMember" + return path + + +class SourceLinkedinPages(AbstractSource): + """ + Abstract Source inheritance, provides: + - implementation for `check` connector's connectivity + - implementation to call each stream with it's input parameters. + """ + + @classmethod + def get_authenticator(cls, config: Mapping[str, Any]) -> TokenAuthenticator: + """ + Validate input parameters and generate a necessary Authentication object + This connectors support 2 auth methods: + 1) direct access token with TTL = 2 months + 2) refresh token (TTL = 1 year) which can be converted to access tokens + Every new refresh revokes all previous access tokens q + """ + auth_method = config.get("credentials", {}).get("auth_method") + if not auth_method or auth_method == "access_token": + # support of backward compatibility with old exists configs + access_token = config["credentials"]["access_token"] if auth_method else config["access_token"] + return TokenAuthenticator(token=access_token) + elif auth_method == "oAuth2.0": + return Oauth2Authenticator( + token_refresh_endpoint="https://www.linkedin.com/oauth/v2/accessToken", + client_id=config["credentials"]["client_id"], + client_secret=config["credentials"]["client_secret"], + refresh_token=config["credentials"]["refresh_token"], + ) + raise Exception("incorrect input parameters") + + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: + # RUN $ python main.py check --config secrets/config.json + + """ + Testing connection availability for the connector. + :: for this check method the Customer must have the "r_liteprofile" scope enabled. + :: more info: https://docs.microsoft.com/linkedin/consumer/integrations/self-serve/sign-in-with-linkedin + """ + + config["authenticator"] = self.get_authenticator(config) + stream = OrganizationLookup(config) + stream.records_limit = 1 + try: + next(stream.read_records(sync_mode=SyncMode.full_refresh), None) + return True, None + except Exception as e: + return False, e + + # RUN: $ python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + config["authenticator"] = self.get_authenticator(config) + return [ + OrganizationLookup(config), + FollowerStatistics(config), + ShareStatistics(config), + TotalFollowerCount(config), + ] diff --git a/source-linkedin-pages/source_linkedin_pages/spec.json b/source-linkedin-pages/source_linkedin_pages/spec.json new file mode 100644 index 0000000000..dfb3a0460e --- /dev/null +++ b/source-linkedin-pages/source_linkedin_pages/spec.json @@ -0,0 +1,71 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/linkedin-pages/", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Linkedin Pages Spec", + "type": "object", + "required": ["org_id"], + "additionalProperties": true, + "properties": { + "org_id": { + "title": "Organization ID", + "type": "string", + "airbyte_secret": true, + "description": "Specify the Organization ID", + "examples": ["123456789"] + }, + "credentials": { + "title": "Authentication", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": ["client_id", "client_secret", "refresh_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "oAuth2.0" + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The client ID of the LinkedIn developer application.", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client secret", + "description": "The client secret of the LinkedIn developer application.", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "title": "Refresh token", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "airbyte_secret": true + } + } + }, + { + "title": "Access token", + "type": "object", + "required": ["access_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + "title": "Access token", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "airbyte_secret": true + } + } + } + ] + } + } + } +} diff --git a/source-linkedin-pages/source_linkedin_pages/utils.py b/source-linkedin-pages/source_linkedin_pages/utils.py new file mode 100644 index 0000000000..3f98d82156 --- /dev/null +++ b/source-linkedin-pages/source_linkedin_pages/utils.py @@ -0,0 +1,324 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from typing import Any, Dict, Iterable, List, Mapping + +import pendulum as pdm + + +def get_parent_stream_values(record: Dict, key_value_map: Dict) -> Dict: + """ + Outputs the Dict with key:value slices for the stream. + :: EXAMPLE: + Input: + records = [{dict}, {dict}, ...], + key_value_map = {: } + + Output: + { + : records..value, + } + """ + result = {} + for key in key_value_map: + value = record.get(key_value_map[key]) + if value: + result[key] = value + return result + + +def transform_change_audit_stamps( + record: Dict, dict_key: str = "changeAuditStamps", props: List = ["created", "lastModified"], fields: List = ["time"] +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `changeAuditStamps` input structure: + { + "changeAuditStamps": { + "created": {"time": 1629581275000}, + "lastModified": {"time": 1629664544760} + } + } + + :: EXAMPLE output: + { + "created": "2021-08-21 21:27:55", + "lastModified": "2021-08-22 20:35:44" + } + """ + + target_dict: Dict = record.get(dict_key) + for prop in props: + # Update dict with flatten key:value + for field in fields: + record[prop] = pdm.from_timestamp(target_dict.get(prop).get(field) / 1000).to_datetime_string() + record.pop(dict_key) + + return record + + +def date_str_from_date_range(record: Dict, prefix: str) -> str: + """ + Makes the ISO8601 format date string from the input . + + EXAMPLE: + Input: record + { + "start.year": 2021, "start.month": 8, "start.day": 1, + "end.year": 2021, "end.month": 9, "end.day": 31 + } + + EXAMPLE output: + With `prefix` = "start" + str: "2021-08-13", + + With `prefix` = "end" + str: "2021-09-31", + """ + + year = record.get(f"{prefix}.year") + month = record.get(f"{prefix}.month") + day = record.get(f"{prefix}.day") + return pdm.date(year, month, day).to_date_string() + + +def transform_date_range( + record: Dict, + dict_key: str = "dateRange", + props: List = ["start", "end"], + fields: List = ["year", "month", "day"], +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `dateRange` input structure in Analytics streams: + { + "dateRange": { + "start": {"month": 8, "day": 13, "year": 2021}, + "end": {"month": 8, "day": 13, "year": 2021} + } + } + :: EXAMPLE output: + { + "start_date": "2021-08-13", + "end_date": "2021-08-13" + } + """ + # define list of tmp keys for cleanup. + keys_to_remove = [dict_key, "start.day", "start.month", "start.year", "end.day", "end.month", "end.year", "start", "end"] + + target_dict: Dict = record.get(dict_key) + for prop in props: + # Update dict with flatten key:value + for field in fields: + record.update(**{f"{prop}.{field}": target_dict.get(prop).get(field)}) + # We build `start_date` & `end_date` fields from nested structure. + record.update(**{"start_date": date_str_from_date_range(record, "start"), "end_date": date_str_from_date_range(record, "end")}) + # Cleanup tmp fields & nested used parts + for key in keys_to_remove: + if key in record: + record.pop(key) + return record + + +def transform_targeting_criteria( + record: Dict, + dict_key: str = "targetingCriteria", +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `targetingCriteria` input structure: + { + "targetingCriteria": { + "include": { + "and": [ + { + "or": { + "urn:li:adTargetingFacet:titles": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ] + } + }, + {"or": {"urn:li:adTargetingFacet:locations": ["urn:li:geo:103644278"]}}, + {"or": {"urn:li:adTargetingFacet:interfaceLocales": ["urn:li:locale:en_US"]}}, + ] + }, + "exclude": { + "or": { + "urn:li:adTargetingFacet:facet_Key1": [ + "facet_test1", + "facet_test2", + ], + "urn:li:adTargetingFacet:facet_Key2": [ + "facet_test3", + "facet_test4", + ], + } + } + } + + :: EXAMPLE output: + { + "targetingCriteria": { + "include": { + "and": [ + { + "type": "urn:li:adTargetingFacet:titles", + "values": [ + "urn:li:title:100", + "urn:li:title:10326", + "urn:li:title:10457", + "urn:li:title:10738", + "urn:li:title:10966", + "urn:li:title:11349", + "urn:li:title:1159", + ], + }, + { + "type": "urn:li:adTargetingFacet:locations", + "values": ["urn:li:geo:103644278"], + }, + { + "type": "urn:li:adTargetingFacet:interfaceLocales", + "values": ["urn:li:locale:en_US"], + }, + ] + }, + "exclude": { + "or": [ + { + "type": "urn:li:adTargetingFacet:facet_Key1", + "values": ["facet_test1", "facet_test2"], + }, + { + "type": "urn:li:adTargetingFacet:facet_Key2", + "values": ["facet_test3", "facet_test4"], + }, + ] + }, + } + + """ + + def unnest_dict(nested_dict: Dict) -> Iterable[Dict]: + """ + Unnest the nested dict to simplify the normalization + + EXAMPLE OUTPUT: + [ + {"type": "some_key", "values": "some_values"}, + ..., + {"type": "some_other_key", "values": "some_other_values"} + ] + """ + + for key, value in nested_dict.items(): + values = [] + if isinstance(value, List): + if len(value) > 0: + if isinstance(value[0], str): + values = value + elif isinstance(value[0], Dict): + for v in value: + values.append(v) + elif isinstance(value, Dict): + values.append(value) + yield {"type": key, "values": values} + + # get the target dict from record + targeting_criteria = record.get(dict_key) + + # transform `include` + if "include" in targeting_criteria: + and_list = targeting_criteria.get("include").get("and") + updated_include = {"and": []} + for k in and_list: + or_dict = k.get("or") + for j in unnest_dict(or_dict): + updated_include["and"].append(j) + # Replace the original 'and' with updated_include + record["targetingCriteria"]["include"] = updated_include + + # transform `exclude` if present + if "exclude" in targeting_criteria: + or_dict = targeting_criteria.get("exclude").get("or") + updated_exclude = {"or": []} + for k in unnest_dict(or_dict): + updated_exclude["or"].append(k) + # Replace the original 'or' with updated_exclude + record["targetingCriteria"]["exclude"] = updated_exclude + + return record + + +def transform_variables( + record: Dict, + dict_key: str = "variables", +) -> Mapping[str, Any]: + + """ + :: EXAMPLE `variables` input: + { + "variables": { + "data": { + "com.linkedin.ads.SponsoredUpdateCreativeVariables": { + "activity": "urn:li:activity:1234", + "directSponsoredContent": 0, + "share": "urn:li:share:1234", + } + } + } + } + + :: EXAMPLE output: + { + "variables": { + "type": "com.linkedin.ads.SponsoredUpdateCreativeVariables", + "values": [ + {"key": "activity", "value": "urn:li:activity:1234"}, + {"key": "directSponsoredContent", "value": 0}, + {"key": "share", "value": "urn:li:share:1234"}, + ], + } + } + """ + + variables = record.get(dict_key).get("data") + for key, params in variables.items(): + record["variables"]["type"] = key + record["variables"]["values"] = [] + for key, value in params.items(): + # convert various datatypes of values into the string + record["variables"]["values"].append({"key": key, "value": json.dumps(value, ensure_ascii=True)}) + # Clean the nested structure + record["variables"].pop("data") + return record + + +def transform_data(records: List) -> Iterable[Mapping]: + """ + We need to transform the nested complex data structures into simple key:value pair, + to be properly normalised in the destination. + """ + for record in records: + + if "changeAuditStamps" in record: + record = transform_change_audit_stamps(record) + + if "dateRange" in record: + record = transform_date_range(record) + + if "targetingCriteria" in record: + record = transform_targeting_criteria(record) + + if "variables" in record: + record = transform_variables(record) + + yield record diff --git a/source-linkedin-pages/unit_tests/__init__.py b/source-linkedin-pages/unit_tests/__init__.py new file mode 100644 index 0000000000..46b7376756 --- /dev/null +++ b/source-linkedin-pages/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# From 30b32eb90311b7d786e849fa6c41d44539ccbd7b Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 28 Mar 2024 13:10:09 -0300 Subject: [PATCH 66/96] source-linkedin-pages: Init poetry project and ditch requirements.txt --- source-linkedin-pages/poetry.lock | 1743 ++++++++++++++++++++++++ source-linkedin-pages/pyproject.toml | 21 + source-linkedin-pages/requirements.txt | 1 - 3 files changed, 1764 insertions(+), 1 deletion(-) create mode 100644 source-linkedin-pages/poetry.lock create mode 100644 source-linkedin-pages/pyproject.toml delete mode 100644 source-linkedin-pages/requirements.txt diff --git a/source-linkedin-pages/poetry.lock b/source-linkedin-pages/poetry.lock new file mode 100644 index 0000000000..fb4742132f --- /dev/null +++ b/source-linkedin-pages/poetry.lock @@ -0,0 +1,1743 @@ +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "aiodns" +version = "3.1.1" +description = "Simple DNS resolver for asyncio" +optional = false +python-versions = "*" +files = [ + {file = "aiodns-3.1.1-py3-none-any.whl", hash = "sha256:a387b63da4ced6aad35b1dda2d09620ad608a1c7c0fb71efa07ebb4cd511928d"}, + {file = "aiodns-3.1.1.tar.gz", hash = "sha256:1073eac48185f7a4150cad7f96a5192d6911f12b4fb894de80a088508c9b3a99"}, +] + +[package.dependencies] +pycares = ">=4.0.0" + +[[package]] +name = "aiohttp" +version = "3.9.3" +description = "Async http client/server framework (asyncio)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, + {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, + {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, + {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, + {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, + {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, + {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, + {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, + {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, + {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, + {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, + {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, +] + +[package.dependencies] +aiosignal = ">=1.1.2" +attrs = ">=17.3.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns", "brotlicffi"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "airbyte-cdk" +version = "0.77.2" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "airbyte_cdk-0.77.2-py3-none-any.whl", hash = "sha256:6dffbe0c4b3454a5cdd20525b4f1e9cfef2e80c005b6b30473fc5bf6f75af64e"}, + {file = "airbyte_cdk-0.77.2.tar.gz", hash = "sha256:84aeb27862a18e135c7bc3a5dfc363037665d428e7495e8824673f853adcca70"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<1.3" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<0.3" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1,<7.0.0" +requests = "*" +requests_cache = "*" +wcmatch = "8.4" + +[package.extras] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.3" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2024.2.2" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "estuary-cdk" +version = "0.2.0" +description = "Estuary Connector Development Kit" +optional = false +python-versions = "^3.11" +files = [] +develop = true + +[package.dependencies] +aiodns = "^3.1.1" +aiohttp = "^3.9.3" +orjson = "^3.9.15" +pydantic = ">1.10,<3" +xxhash = "^3.4.1" + +[package.source] +type = "directory" +url = "../estuary-cdk" + +[[package]] +name = "frozenlist" +version = "1.4.1" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = false +python-versions = ">=3.8" +files = [ + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc"}, + {file = "frozenlist-1.4.1-cp310-cp310-win32.whl", hash = "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1"}, + {file = "frozenlist-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"}, + {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"}, + {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8"}, + {file = "frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89"}, + {file = "frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7"}, + {file = "frozenlist-1.4.1-cp38-cp38-win32.whl", hash = "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497"}, + {file = "frozenlist-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6"}, + {file = "frozenlist-1.4.1-cp39-cp39-win32.whl", hash = "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932"}, + {file = "frozenlist-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0"}, + {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"}, + {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, +] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[package.extras] +colors = ["colorama (>=0.4.6)"] + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.2" +description = "An implementation of JSON Reference for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonref-0.2-py3-none-any.whl", hash = "sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f"}, + {file = "jsonref-0.2.tar.gz", hash = "sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "multidict" +version = "6.0.5" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, +] + +[[package]] +name = "orjson" +version = "3.10.0" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c90681333619d78360d13840c7235fdaf01b2b129cb3a4f1647783b1971542b6"}, + {file = "orjson-3.10.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:400c5b7c4222cb27b5059adf1fb12302eebcabf1978f33d0824aa5277ca899bd"}, + {file = "orjson-3.10.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5dcb32e949eae80fb335e63b90e5808b4b0f64e31476b3777707416b41682db5"}, + {file = "orjson-3.10.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa7d507c7493252c0a0264b5cc7e20fa2f8622b8a83b04d819b5ce32c97cf57b"}, + {file = "orjson-3.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e286a51def6626f1e0cc134ba2067dcf14f7f4b9550f6dd4535fd9d79000040b"}, + {file = "orjson-3.10.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8acd4b82a5f3a3ec8b1dc83452941d22b4711964c34727eb1e65449eead353ca"}, + {file = "orjson-3.10.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:30707e646080dd3c791f22ce7e4a2fc2438765408547c10510f1f690bd336217"}, + {file = "orjson-3.10.0-cp310-none-win32.whl", hash = "sha256:115498c4ad34188dcb73464e8dc80e490a3e5e88a925907b6fedcf20e545001a"}, + {file = "orjson-3.10.0-cp310-none-win_amd64.whl", hash = "sha256:6735dd4a5a7b6df00a87d1d7a02b84b54d215fb7adac50dd24da5997ffb4798d"}, + {file = "orjson-3.10.0-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9587053e0cefc284e4d1cd113c34468b7d3f17666d22b185ea654f0775316a26"}, + {file = "orjson-3.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bef1050b1bdc9ea6c0d08468e3e61c9386723633b397e50b82fda37b3563d72"}, + {file = "orjson-3.10.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d16c6963ddf3b28c0d461641517cd312ad6b3cf303d8b87d5ef3fa59d6844337"}, + {file = "orjson-3.10.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4251964db47ef090c462a2d909f16c7c7d5fe68e341dabce6702879ec26d1134"}, + {file = "orjson-3.10.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73bbbdc43d520204d9ef0817ac03fa49c103c7f9ea94f410d2950755be2c349c"}, + {file = "orjson-3.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:414e5293b82373606acf0d66313aecb52d9c8c2404b1900683eb32c3d042dbd7"}, + {file = "orjson-3.10.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:feaed5bb09877dc27ed0d37f037ddef6cb76d19aa34b108db270d27d3d2ef747"}, + {file = "orjson-3.10.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5127478260db640323cea131ee88541cb1a9fbce051f0b22fa2f0892f44da302"}, + {file = "orjson-3.10.0-cp311-none-win32.whl", hash = "sha256:b98345529bafe3c06c09996b303fc0a21961820d634409b8639bc16bd4f21b63"}, + {file = "orjson-3.10.0-cp311-none-win_amd64.whl", hash = "sha256:658ca5cee3379dd3d37dbacd43d42c1b4feee99a29d847ef27a1cb18abdfb23f"}, + {file = "orjson-3.10.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4329c1d24fd130ee377e32a72dc54a3c251e6706fccd9a2ecb91b3606fddd998"}, + {file = "orjson-3.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef0f19fdfb6553342b1882f438afd53c7cb7aea57894c4490c43e4431739c700"}, + {file = "orjson-3.10.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c4f60db24161534764277f798ef53b9d3063092f6d23f8f962b4a97edfa997a0"}, + {file = "orjson-3.10.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1de3fd5c7b208d836f8ecb4526995f0d5877153a4f6f12f3e9bf11e49357de98"}, + {file = "orjson-3.10.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f93e33f67729d460a177ba285002035d3f11425ed3cebac5f6ded4ef36b28344"}, + {file = "orjson-3.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:237ba922aef472761acd697eef77fef4831ab769a42e83c04ac91e9f9e08fa0e"}, + {file = "orjson-3.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98c1bfc6a9bec52bc8f0ab9b86cc0874b0299fccef3562b793c1576cf3abb570"}, + {file = "orjson-3.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:30d795a24be16c03dca0c35ca8f9c8eaaa51e3342f2c162d327bd0225118794a"}, + {file = "orjson-3.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ade1e21dfde1d37feee8cf6464c20a2f41fa46c8bcd5251e761903e46102dc6b"}, + {file = "orjson-3.10.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:23c12bb4ced1c3308eff7ba5c63ef8f0edb3e4c43c026440247dd6c1c61cea4b"}, + {file = "orjson-3.10.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2d014cf8d4dc9f03fc9f870de191a49a03b1bcda51f2a957943fb9fafe55aac"}, + {file = "orjson-3.10.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eadecaa16d9783affca33597781328e4981b048615c2ddc31c47a51b833d6319"}, + {file = "orjson-3.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd583341218826f48bd7c6ebf3310b4126216920853cbc471e8dbeaf07b0b80e"}, + {file = "orjson-3.10.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:90bfc137c75c31d32308fd61951d424424426ddc39a40e367704661a9ee97095"}, + {file = "orjson-3.10.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:13b5d3c795b09a466ec9fcf0bd3ad7b85467d91a60113885df7b8d639a9d374b"}, + {file = "orjson-3.10.0-cp38-none-win32.whl", hash = "sha256:5d42768db6f2ce0162544845facb7c081e9364a5eb6d2ef06cd17f6050b048d8"}, + {file = "orjson-3.10.0-cp38-none-win_amd64.whl", hash = "sha256:33e6655a2542195d6fd9f850b428926559dee382f7a862dae92ca97fea03a5ad"}, + {file = "orjson-3.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1897aa25a944cec774ce4a0e1c8e98fb50523e97366c637b7d0cddabc42e6643"}, + {file = "orjson-3.10.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9bf565a69e0082ea348c5657401acec3cbbb31564d89afebaee884614fba36b4"}, + {file = "orjson-3.10.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b6ebc17cfbbf741f5c1a888d1854354536f63d84bee537c9a7c0335791bb9009"}, + {file = "orjson-3.10.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2817877d0b69f78f146ab305c5975d0618df41acf8811249ee64231f5953fee"}, + {file = "orjson-3.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57d017863ec8aa4589be30a328dacd13c2dc49de1c170bc8d8c8a98ece0f2925"}, + {file = "orjson-3.10.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:22c2f7e377ac757bd3476ecb7480c8ed79d98ef89648f0176deb1da5cd014eb7"}, + {file = "orjson-3.10.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e62ba42bfe64c60c1bc84799944f80704e996592c6b9e14789c8e2a303279912"}, + {file = "orjson-3.10.0-cp39-none-win32.whl", hash = "sha256:60c0b1bdbccd959ebd1575bd0147bd5e10fc76f26216188be4a36b691c937077"}, + {file = "orjson-3.10.0-cp39-none-win_amd64.whl", hash = "sha256:175a41500ebb2fdf320bf78e8b9a75a1279525b62ba400b2b2444e274c2c8bee"}, + {file = "orjson-3.10.0.tar.gz", hash = "sha256:ba4d8cac5f2e2cff36bea6b6481cdb92b38c202bcec603d6f5ff91960595a1ed"}, +] + +[[package]] +name = "packaging" +version = "24.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pycares" +version = "4.4.0" +description = "Python interface for c-ares" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycares-4.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:24da119850841d16996713d9c3374ca28a21deee056d609fbbed29065d17e1f6"}, + {file = "pycares-4.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8f64cb58729689d4d0e78f0bfb4c25ce2f851d0274c0273ac751795c04b8798a"}, + {file = "pycares-4.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d33e2a1120887e89075f7f814ec144f66a6ce06a54f5722ccefc62fbeda83cff"}, + {file = "pycares-4.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c680fef1b502ee680f8f0b95a41af4ec2c234e50e16c0af5bbda31999d3584bd"}, + {file = "pycares-4.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fff16b09042ba077f7b8aa5868d1d22456f0002574d0ba43462b10a009331677"}, + {file = "pycares-4.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:229a1675eb33bc9afb1fc463e73ee334950ccc485bc83a43f6ae5839fb4d5fa3"}, + {file = "pycares-4.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3aebc73e5ad70464f998f77f2da2063aa617cbd8d3e8174dd7c5b4518f967153"}, + {file = "pycares-4.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6ef64649eba56448f65e26546d85c860709844d2fc22ef14d324fe0b27f761a9"}, + {file = "pycares-4.4.0-cp310-cp310-win32.whl", hash = "sha256:4afc2644423f4eef97857a9fd61be9758ce5e336b4b0bd3d591238bb4b8b03e0"}, + {file = "pycares-4.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:5ed4e04af4012f875b78219d34434a6d08a67175150ac1b79eb70ab585d4ba8c"}, + {file = "pycares-4.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bce8db2fc6f3174bd39b81405210b9b88d7b607d33e56a970c34a0c190da0490"}, + {file = "pycares-4.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a0303428d013ccf5c51de59c83f9127aba6200adb7fd4be57eddb432a1edd2a"}, + {file = "pycares-4.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afb91792f1556f97be7f7acb57dc7756d89c5a87bd8b90363a77dbf9ea653817"}, + {file = "pycares-4.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b61579cecf1f4d616e5ea31a6e423a16680ab0d3a24a2ffe7bb1d4ee162477ff"}, + {file = "pycares-4.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7af06968cbf6851566e806bf3e72825b0e6671832a2cbe840be1d2d65350710"}, + {file = "pycares-4.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ceb12974367b0a68a05d52f4162b29f575d241bd53de155efe632bf2c943c7f6"}, + {file = "pycares-4.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2eeec144bcf6a7b6f2d74d6e70cbba7886a84dd373c886f06cb137a07de4954c"}, + {file = "pycares-4.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e3a6f7cfdfd11eb5493d6d632e582408c8f3b429f295f8799c584c108b28db6f"}, + {file = "pycares-4.4.0-cp311-cp311-win32.whl", hash = "sha256:34736a2ffaa9c08ca9c707011a2d7b69074bbf82d645d8138bba771479b2362f"}, + {file = "pycares-4.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:eb66c30eb11e877976b7ead13632082a8621df648c408b8e15cdb91a452dd502"}, + {file = "pycares-4.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:fd644505a8cfd7f6584d33a9066d4e3d47700f050ef1490230c962de5dfb28c6"}, + {file = "pycares-4.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52084961262232ec04bd75f5043aed7e5d8d9695e542ff691dfef0110209f2d4"}, + {file = "pycares-4.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0c5368206057884cde18602580083aeaad9b860e2eac14fd253543158ce1e93"}, + {file = "pycares-4.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:112a4979c695b1c86f6782163d7dec58d57a3b9510536dcf4826550f9053dd9a"}, + {file = "pycares-4.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d186dafccdaa3409194c0f94db93c1a5d191145a275f19da6591f9499b8e7b8"}, + {file = "pycares-4.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:64965dc19c578a683ea73487a215a8897276224e004d50eeb21f0bc7a0b63c88"}, + {file = "pycares-4.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:ed2a38e34bec6f2586435f6ff0bc5fe11d14bebd7ed492cf739a424e81681540"}, + {file = "pycares-4.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:94d6962db81541eb0396d2f0dfcbb18cdb8c8b251d165efc2d974ae652c547d4"}, + {file = "pycares-4.4.0-cp312-cp312-win32.whl", hash = "sha256:1168a48a834813aa80f412be2df4abaf630528a58d15c704857448b20b1675c0"}, + {file = "pycares-4.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:db24c4e7fea4a052c6e869cbf387dd85d53b9736cfe1ef5d8d568d1ca925e977"}, + {file = "pycares-4.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:21a5a0468861ec7df7befa69050f952da13db5427ae41ffe4713bc96291d1d95"}, + {file = "pycares-4.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:22c00bf659a9fa44d7b405cf1cd69b68b9d37537899898d8cbe5dffa4016b273"}, + {file = "pycares-4.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23aa3993a352491a47fcf17867f61472f32f874df4adcbb486294bd9fbe8abee"}, + {file = "pycares-4.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:813d661cbe2e37d87da2d16b7110a6860e93ddb11735c6919c8a3545c7b9c8d8"}, + {file = "pycares-4.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:77cf5a2fd5583c670de41a7f4a7b46e5cbabe7180d8029f728571f4d2e864084"}, + {file = "pycares-4.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3eaa6681c0a3e3f3868c77aca14b7760fed35fdfda2fe587e15c701950e7bc69"}, + {file = "pycares-4.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ad58e284a658a8a6a84af2e0b62f2f961f303cedfe551854d7bd40c3cbb61912"}, + {file = "pycares-4.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bfb89ca9e3d0a9b5332deeb666b2ede9d3469107742158f4aeda5ce032d003f4"}, + {file = "pycares-4.4.0-cp38-cp38-win32.whl", hash = "sha256:f36bdc1562142e3695555d2f4ac0cb69af165eddcefa98efc1c79495b533481f"}, + {file = "pycares-4.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:902461a92b6a80fd5041a2ec5235680c7cc35e43615639ec2a40e63fca2dfb51"}, + {file = "pycares-4.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7bddc6adba8f699728f7fc1c9ce8cef359817ad78e2ed52b9502cb5f8dc7f741"}, + {file = "pycares-4.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cb49d5805cd347c404f928c5ae7c35e86ba0c58ffa701dbe905365e77ce7d641"}, + {file = "pycares-4.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56cf3349fa3a2e67ed387a7974c11d233734636fe19facfcda261b411af14d80"}, + {file = "pycares-4.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bf2eaa83a5987e48fa63302f0fe7ce3275cfda87b34d40fef9ce703fb3ac002"}, + {file = "pycares-4.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82bba2ab77eb5addbf9758d514d9bdef3c1bfe7d1649a47bd9a0d55a23ef478b"}, + {file = "pycares-4.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c6a8bde63106f162fca736e842a916853cad3c8d9d137e11c9ffa37efa818b02"}, + {file = "pycares-4.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f5f646eec041db6ffdbcaf3e0756fb92018f7af3266138c756bb09d2b5baadec"}, + {file = "pycares-4.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9dc04c54c6ea615210c1b9e803d0e2d2255f87a3d5d119b6482c8f0dfa15b26b"}, + {file = "pycares-4.4.0-cp39-cp39-win32.whl", hash = "sha256:97892cced5794d721fb4ff8765764aa4ea48fe8b2c3820677505b96b83d4ef47"}, + {file = "pycares-4.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:917f08f0b5d9324e9a34211e68d27447c552b50ab967044776bbab7e42a553a2"}, + {file = "pycares-4.4.0.tar.gz", hash = "sha256:f47579d508f2f56eddd16ce72045782ad3b1b3b678098699e2b6a1b30733e1c2"}, +] + +[package.dependencies] +cffi = ">=1.5.0" + +[package.extras] +idna = ["idna (>=2.1)"] + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.10.14" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, + {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, + {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, + {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, + {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, + {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, + {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, + {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, + {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, + {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, + {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, + {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, + {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, + {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, + {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, + {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, + {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, + {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, + {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, + {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, + {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, + {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, + {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "8.1.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"}, + {file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.4,<2.0" + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-snapshot" +version = "0.9.0" +description = "A plugin for snapshot testing with pytest." +optional = false +python-versions = ">=3.5" +files = [ + {file = "pytest-snapshot-0.9.0.tar.gz", hash = "sha256:c7013c3abc3e860f9feff899f8b4debe3708650d8d8242a61bf2625ff64db7f3"}, + {file = "pytest_snapshot-0.9.0-py3-none-any.whl", hash = "sha256:4b9fe1c21c868fe53a545e4e3184d36bc1c88946e3f5c1d9dd676962a9b3d4ab"}, +] + +[package.dependencies] +pytest = ">=3.0.0" + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.2.0" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "ruff" +version = "0.3.4" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.3.4-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:60c870a7d46efcbc8385d27ec07fe534ac32f3b251e4fc44b3cbfd9e09609ef4"}, + {file = "ruff-0.3.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6fc14fa742e1d8f24910e1fff0bd5e26d395b0e0e04cc1b15c7c5e5fe5b4af91"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3ee7880f653cc03749a3bfea720cf2a192e4f884925b0cf7eecce82f0ce5854"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf133dd744f2470b347f602452a88e70dadfbe0fcfb5fd46e093d55da65f82f7"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f3860057590e810c7ffea75669bdc6927bfd91e29b4baa9258fd48b540a4365"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:986f2377f7cf12efac1f515fc1a5b753c000ed1e0a6de96747cdf2da20a1b369"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fd98e85869603e65f554fdc5cddf0712e352fe6e61d29d5a6fe087ec82b76c"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64abeed785dad51801b423fa51840b1764b35d6c461ea8caef9cf9e5e5ab34d9"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df52972138318bc7546d92348a1ee58449bc3f9eaf0db278906eb511889c4b50"}, + {file = "ruff-0.3.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:98e98300056445ba2cc27d0b325fd044dc17fcc38e4e4d2c7711585bd0a958ed"}, + {file = "ruff-0.3.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:519cf6a0ebed244dce1dc8aecd3dc99add7a2ee15bb68cf19588bb5bf58e0488"}, + {file = "ruff-0.3.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bb0acfb921030d00070539c038cd24bb1df73a2981e9f55942514af8b17be94e"}, + {file = "ruff-0.3.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cf187a7e7098233d0d0c71175375c5162f880126c4c716fa28a8ac418dcf3378"}, + {file = "ruff-0.3.4-py3-none-win32.whl", hash = "sha256:af27ac187c0a331e8ef91d84bf1c3c6a5dea97e912a7560ac0cef25c526a4102"}, + {file = "ruff-0.3.4-py3-none-win_amd64.whl", hash = "sha256:de0d5069b165e5a32b3c6ffbb81c350b1e3d3483347196ffdf86dc0ef9e37dd6"}, + {file = "ruff-0.3.4-py3-none-win_arm64.whl", hash = "sha256:6810563cc08ad0096b57c717bd78aeac888a1bfd38654d9113cb3dc4d3f74232"}, + {file = "ruff-0.3.4.tar.gz", hash = "sha256:f0f4484c6541a99862b693e13a151435a279b271cff20e37101116a21e2a1ad1"}, +] + +[[package]] +name = "setuptools" +version = "69.2.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, + {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "typing-extensions" +version = "4.10.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.2.1" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "xxhash" +version = "3.4.1" +description = "Python binding for xxHash" +optional = false +python-versions = ">=3.7" +files = [ + {file = "xxhash-3.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:91dbfa55346ad3e18e738742236554531a621042e419b70ad8f3c1d9c7a16e7f"}, + {file = "xxhash-3.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:665a65c2a48a72068fcc4d21721510df5f51f1142541c890491afc80451636d2"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb11628470a6004dc71a09fe90c2f459ff03d611376c1debeec2d648f44cb693"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5bef2a7dc7b4f4beb45a1edbba9b9194c60a43a89598a87f1a0226d183764189"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c0f7b2d547d72c7eda7aa817acf8791f0146b12b9eba1d4432c531fb0352228"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00f2fdef6b41c9db3d2fc0e7f94cb3db86693e5c45d6de09625caad9a469635b"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23cfd9ca09acaf07a43e5a695143d9a21bf00f5b49b15c07d5388cadf1f9ce11"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6a9ff50a3cf88355ca4731682c168049af1ca222d1d2925ef7119c1a78e95b3b"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f1d7c69a1e9ca5faa75546fdd267f214f63f52f12692f9b3a2f6467c9e67d5e7"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:672b273040d5d5a6864a36287f3514efcd1d4b1b6a7480f294c4b1d1ee1b8de0"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4178f78d70e88f1c4a89ff1ffe9f43147185930bb962ee3979dba15f2b1cc799"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9804b9eb254d4b8cc83ab5a2002128f7d631dd427aa873c8727dba7f1f0d1c2b"}, + {file = "xxhash-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c09c49473212d9c87261d22c74370457cfff5db2ddfc7fd1e35c80c31a8c14ce"}, + {file = "xxhash-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ebbb1616435b4a194ce3466d7247df23499475c7ed4eb2681a1fa42ff766aff6"}, + {file = "xxhash-3.4.1-cp310-cp310-win_arm64.whl", hash = "sha256:25dc66be3db54f8a2d136f695b00cfe88018e59ccff0f3b8f545869f376a8a46"}, + {file = "xxhash-3.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:58c49083801885273e262c0f5bbeac23e520564b8357fbb18fb94ff09d3d3ea5"}, + {file = "xxhash-3.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b526015a973bfbe81e804a586b703f163861da36d186627e27524f5427b0d520"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36ad4457644c91a966f6fe137d7467636bdc51a6ce10a1d04f365c70d6a16d7e"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:248d3e83d119770f96003271fe41e049dd4ae52da2feb8f832b7a20e791d2920"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2070b6d5bbef5ee031666cf21d4953c16e92c2f8a24a94b5c240f8995ba3b1d0"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2746035f518f0410915e247877f7df43ef3372bf36cfa52cc4bc33e85242641"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a8ba6181514681c2591840d5632fcf7356ab287d4aff1c8dea20f3c78097088"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0aac5010869240e95f740de43cd6a05eae180c59edd182ad93bf12ee289484fa"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4cb11d8debab1626181633d184b2372aaa09825bde709bf927704ed72765bed1"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b29728cff2c12f3d9f1d940528ee83918d803c0567866e062683f300d1d2eff3"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a15cbf3a9c40672523bdb6ea97ff74b443406ba0ab9bca10ceccd9546414bd84"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6e66df260fed01ed8ea790c2913271641c58481e807790d9fca8bfd5a3c13844"}, + {file = "xxhash-3.4.1-cp311-cp311-win32.whl", hash = "sha256:e867f68a8f381ea12858e6d67378c05359d3a53a888913b5f7d35fbf68939d5f"}, + {file = "xxhash-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:200a5a3ad9c7c0c02ed1484a1d838b63edcf92ff538770ea07456a3732c577f4"}, + {file = "xxhash-3.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:1d03f1c0d16d24ea032e99f61c552cb2b77d502e545187338bea461fde253583"}, + {file = "xxhash-3.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c4bbba9b182697a52bc0c9f8ec0ba1acb914b4937cd4a877ad78a3b3eeabefb3"}, + {file = "xxhash-3.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9fd28a9da300e64e434cfc96567a8387d9a96e824a9be1452a1e7248b7763b78"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6066d88c9329ab230e18998daec53d819daeee99d003955c8db6fc4971b45ca3"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93805bc3233ad89abf51772f2ed3355097a5dc74e6080de19706fc447da99cd3"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64da57d5ed586ebb2ecdde1e997fa37c27fe32fe61a656b77fabbc58e6fbff6e"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a97322e9a7440bf3c9805cbaac090358b43f650516486746f7fa482672593df"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbe750d512982ee7d831838a5dee9e9848f3fb440e4734cca3f298228cc957a6"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:fd79d4087727daf4d5b8afe594b37d611ab95dc8e29fe1a7517320794837eb7d"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:743612da4071ff9aa4d055f3f111ae5247342931dedb955268954ef7201a71ff"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:b41edaf05734092f24f48c0958b3c6cbaaa5b7e024880692078c6b1f8247e2fc"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:a90356ead70d715fe64c30cd0969072de1860e56b78adf7c69d954b43e29d9fa"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac56eebb364e44c85e1d9e9cc5f6031d78a34f0092fea7fc80478139369a8b4a"}, + {file = "xxhash-3.4.1-cp312-cp312-win32.whl", hash = "sha256:911035345932a153c427107397c1518f8ce456f93c618dd1c5b54ebb22e73747"}, + {file = "xxhash-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:f31ce76489f8601cc7b8713201ce94b4bd7b7ce90ba3353dccce7e9e1fee71fa"}, + {file = "xxhash-3.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:b5beb1c6a72fdc7584102f42c4d9df232ee018ddf806e8c90906547dfb43b2da"}, + {file = "xxhash-3.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6d42b24d1496deb05dee5a24ed510b16de1d6c866c626c2beb11aebf3be278b9"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b685fab18876b14a8f94813fa2ca80cfb5ab6a85d31d5539b7cd749ce9e3624"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:419ffe34c17ae2df019a4685e8d3934d46b2e0bbe46221ab40b7e04ed9f11137"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e041ce5714f95251a88670c114b748bca3bf80cc72400e9f23e6d0d59cf2681"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc860d887c5cb2f524899fb8338e1bb3d5789f75fac179101920d9afddef284b"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:312eba88ffe0a05e332e3a6f9788b73883752be63f8588a6dc1261a3eaaaf2b2"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:e01226b6b6a1ffe4e6bd6d08cfcb3ca708b16f02eb06dd44f3c6e53285f03e4f"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9f3025a0d5d8cf406a9313cd0d5789c77433ba2004b1c75439b67678e5136537"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:6d3472fd4afef2a567d5f14411d94060099901cd8ce9788b22b8c6f13c606a93"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:43984c0a92f06cac434ad181f329a1445017c33807b7ae4f033878d860a4b0f2"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a55e0506fdb09640a82ec4f44171273eeabf6f371a4ec605633adb2837b5d9d5"}, + {file = "xxhash-3.4.1-cp37-cp37m-win32.whl", hash = "sha256:faec30437919555b039a8bdbaba49c013043e8f76c999670aef146d33e05b3a0"}, + {file = "xxhash-3.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:c9e1b646af61f1fc7083bb7b40536be944f1ac67ef5e360bca2d73430186971a"}, + {file = "xxhash-3.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:961d948b7b1c1b6c08484bbce3d489cdf153e4122c3dfb07c2039621243d8795"}, + {file = "xxhash-3.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:719a378930504ab159f7b8e20fa2aa1896cde050011af838af7e7e3518dd82de"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74fb5cb9406ccd7c4dd917f16630d2e5e8cbbb02fc2fca4e559b2a47a64f4940"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5dab508ac39e0ab988039bc7f962c6ad021acd81fd29145962b068df4148c476"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c59f3e46e7daf4c589e8e853d700ef6607afa037bfad32c390175da28127e8c"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cc07256eff0795e0f642df74ad096f8c5d23fe66bc138b83970b50fc7f7f6c5"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9f749999ed80f3955a4af0eb18bb43993f04939350b07b8dd2f44edc98ffee9"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7688d7c02149a90a3d46d55b341ab7ad1b4a3f767be2357e211b4e893efbaaf6"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a8b4977963926f60b0d4f830941c864bed16aa151206c01ad5c531636da5708e"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:8106d88da330f6535a58a8195aa463ef5281a9aa23b04af1848ff715c4398fb4"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4c76a77dbd169450b61c06fd2d5d436189fc8ab7c1571d39265d4822da16df22"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:11f11357c86d83e53719c592021fd524efa9cf024dc7cb1dfb57bbbd0d8713f2"}, + {file = "xxhash-3.4.1-cp38-cp38-win32.whl", hash = "sha256:0c786a6cd74e8765c6809892a0d45886e7c3dc54de4985b4a5eb8b630f3b8e3b"}, + {file = "xxhash-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:aabf37fb8fa27430d50507deeab2ee7b1bcce89910dd10657c38e71fee835594"}, + {file = "xxhash-3.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6127813abc1477f3a83529b6bbcfeddc23162cece76fa69aee8f6a8a97720562"}, + {file = "xxhash-3.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef2e194262f5db16075caea7b3f7f49392242c688412f386d3c7b07c7733a70a"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71be94265b6c6590f0018bbf73759d21a41c6bda20409782d8117e76cd0dfa8b"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10e0a619cdd1c0980e25eb04e30fe96cf8f4324758fa497080af9c21a6de573f"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa122124d2e3bd36581dd78c0efa5f429f5220313479fb1072858188bc2d5ff1"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17032f5a4fea0a074717fe33477cb5ee723a5f428de7563e75af64bfc1b1e10"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca7783b20e3e4f3f52f093538895863f21d18598f9a48211ad757680c3bd006f"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d77d09a1113899fad5f354a1eb4f0a9afcf58cefff51082c8ad643ff890e30cf"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:21287bcdd299fdc3328cc0fbbdeaa46838a1c05391264e51ddb38a3f5b09611f"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:dfd7a6cc483e20b4ad90224aeb589e64ec0f31e5610ab9957ff4314270b2bf31"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:543c7fcbc02bbb4840ea9915134e14dc3dc15cbd5a30873a7a5bf66039db97ec"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fe0a98d990e433013f41827b62be9ab43e3cf18e08b1483fcc343bda0d691182"}, + {file = "xxhash-3.4.1-cp39-cp39-win32.whl", hash = "sha256:b9097af00ebf429cc7c0e7d2fdf28384e4e2e91008130ccda8d5ae653db71e54"}, + {file = "xxhash-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:d699b921af0dcde50ab18be76c0d832f803034d80470703700cb7df0fbec2832"}, + {file = "xxhash-3.4.1-cp39-cp39-win_arm64.whl", hash = "sha256:2be491723405e15cc099ade1280133ccfbf6322d2ef568494fb7d07d280e7eee"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:431625fad7ab5649368c4849d2b49a83dc711b1f20e1f7f04955aab86cd307bc"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc6dbd5fc3c9886a9e041848508b7fb65fd82f94cc793253990f81617b61fe49"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3ff8dbd0ec97aec842476cb8ccc3e17dd288cd6ce3c8ef38bff83d6eb927817"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef73a53fe90558a4096e3256752268a8bdc0322f4692ed928b6cd7ce06ad4fe3"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:450401f42bbd274b519d3d8dcf3c57166913381a3d2664d6609004685039f9d3"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a162840cf4de8a7cd8720ff3b4417fbc10001eefdd2d21541a8226bb5556e3bb"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b736a2a2728ba45017cb67785e03125a79d246462dfa892d023b827007412c52"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d0ae4c2e7698adef58710d6e7a32ff518b66b98854b1c68e70eee504ad061d8"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6322c4291c3ff174dcd104fae41500e75dad12be6f3085d119c2c8a80956c51"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:dd59ed668801c3fae282f8f4edadf6dc7784db6d18139b584b6d9677ddde1b6b"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92693c487e39523a80474b0394645b393f0ae781d8db3474ccdcead0559ccf45"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4603a0f642a1e8d7f3ba5c4c25509aca6a9c1cc16f85091004a7028607ead663"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa45e8cbfbadb40a920fe9ca40c34b393e0b067082d94006f7f64e70c7490a6"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:595b252943b3552de491ff51e5bb79660f84f033977f88f6ca1605846637b7c6"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:562d8b8f783c6af969806aaacf95b6c7b776929ae26c0cd941d54644ea7ef51e"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:41ddeae47cf2828335d8d991f2d2b03b0bdc89289dc64349d712ff8ce59d0647"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c44d584afdf3c4dbb3277e32321d1a7b01d6071c1992524b6543025fb8f4206f"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd7bddb3a5b86213cc3f2c61500c16945a1b80ecd572f3078ddbbe68f9dabdfb"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ecb6c987b62437c2f99c01e97caf8d25660bf541fe79a481d05732e5236719c"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:696b4e18b7023527d5c50ed0626ac0520edac45a50ec7cf3fc265cd08b1f4c03"}, + {file = "xxhash-3.4.1.tar.gz", hash = "sha256:0379d6cf1ff987cd421609a264ce025e74f346e3e145dd106c0cc2e3ec3f99a9"}, +] + +[[package]] +name = "yarl" +version = "1.9.4" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "e5c243f78a01783701e27af42c46894d2e8df86d3a10a8582f2809bb65f81284" diff --git a/source-linkedin-pages/pyproject.toml b/source-linkedin-pages/pyproject.toml new file mode 100644 index 0000000000..987a1eecd8 --- /dev/null +++ b/source-linkedin-pages/pyproject.toml @@ -0,0 +1,21 @@ +[tool.poetry] +name = "source-linkedin-pages" +version = "0.1.0" +description = "" +authors = ["Pedro "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.11" +estuary-cdk = { path = "../estuary-cdk", develop = true } +airbyte-cdk = "^0.77.2" + +[tool.poetry.group.dev.dependencies] +pytest = "^8.1.1" +pytest-snapshot = "^0.9.0" +ruff = "^0.3.4" +isort = "^5.13.2" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/source-linkedin-pages/requirements.txt b/source-linkedin-pages/requirements.txt deleted file mode 100644 index d6e1198b1a..0000000000 --- a/source-linkedin-pages/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . From d8eee66b2ffa012f9ca05442e8d9aedccca69ec0 Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 28 Mar 2024 13:11:42 -0300 Subject: [PATCH 67/96] source-linkedin-pages: Adapt and refactor Python code source-linkedin-pages: Fix schema null types source-linkedin-pages: Refactor LinkedIn API paths in source.py source-linkedin-pages: Format code with ruff and isort --- source-linkedin-pages/.dockerignore | 6 - source-linkedin-pages/Dockerfile | 38 -- source-linkedin-pages/README.md | 100 ----- .../acceptance-test-config.yml | 29 -- source-linkedin-pages/bootstrap.md | 7 - source-linkedin-pages/icon.svg | 1 - .../integration_tests/__init__.py | 3 - .../integration_tests/abnormal_state.json | 23 -- .../integration_tests/acceptance.py | 13 - .../integration_tests/configured_catalog.json | 40 -- .../integration_tests/invalid_config.json | 7 - .../integration_tests/sample_config.json | 7 - .../integration_tests/sample_state.json | 5 - source-linkedin-pages/main.py | 8 - source-linkedin-pages/metadata.yaml | 29 -- source-linkedin-pages/setup.py | 47 --- .../source_linkedin_pages/__main__.py | 13 + .../source_linkedin_pages/run.py | 14 - .../schemas/follower_statistics.json | 77 +++- .../schemas/organization_lookup.json | 369 ++++++++++++++---- .../schemas/share_statistics.json | 59 ++- .../schemas/total_follower_count.json | 9 +- .../source_linkedin_pages/source.py | 90 ++++- .../source_linkedin_pages/utils.py | 37 +- source-linkedin-pages/unit_tests/__init__.py | 3 - 25 files changed, 517 insertions(+), 517 deletions(-) delete mode 100644 source-linkedin-pages/.dockerignore delete mode 100644 source-linkedin-pages/Dockerfile delete mode 100644 source-linkedin-pages/README.md delete mode 100644 source-linkedin-pages/acceptance-test-config.yml delete mode 100644 source-linkedin-pages/bootstrap.md delete mode 100644 source-linkedin-pages/icon.svg delete mode 100644 source-linkedin-pages/integration_tests/__init__.py delete mode 100644 source-linkedin-pages/integration_tests/abnormal_state.json delete mode 100644 source-linkedin-pages/integration_tests/acceptance.py delete mode 100644 source-linkedin-pages/integration_tests/configured_catalog.json delete mode 100644 source-linkedin-pages/integration_tests/invalid_config.json delete mode 100644 source-linkedin-pages/integration_tests/sample_config.json delete mode 100644 source-linkedin-pages/integration_tests/sample_state.json delete mode 100644 source-linkedin-pages/main.py delete mode 100644 source-linkedin-pages/metadata.yaml delete mode 100644 source-linkedin-pages/setup.py create mode 100644 source-linkedin-pages/source_linkedin_pages/__main__.py delete mode 100644 source-linkedin-pages/source_linkedin_pages/run.py delete mode 100644 source-linkedin-pages/unit_tests/__init__.py diff --git a/source-linkedin-pages/.dockerignore b/source-linkedin-pages/.dockerignore deleted file mode 100644 index 53de6536d1..0000000000 --- a/source-linkedin-pages/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_linkedin_pages -!setup.py -!secrets diff --git a/source-linkedin-pages/Dockerfile b/source-linkedin-pages/Dockerfile deleted file mode 100644 index 674e988746..0000000000 --- a/source-linkedin-pages/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_linkedin_pages ./source_linkedin_pages - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.0.2 -LABEL io.airbyte.name=airbyte/source-linkedin-pages diff --git a/source-linkedin-pages/README.md b/source-linkedin-pages/README.md deleted file mode 100644 index aa8934c3e3..0000000000 --- a/source-linkedin-pages/README.md +++ /dev/null @@ -1,100 +0,0 @@ -# Linkedin Pages Source - -This is the repository for the Linkedin Pages source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/linkedin-pages). - -## Local development - -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linkedin-pages) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linkedin_pages/spec.json` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source linkedin-pages test creds` -and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - -### Locally running the connector docker image - - -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** -```bash -airbyte-ci connectors --name=source-linkedin-pages build -``` - -An image will be built with the tag `airbyte/source-linkedin-pages:dev`. - -**Via `docker build`:** -```bash -docker build -t airbyte/source-linkedin-pages:dev . -``` - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-linkedin-pages:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-linkedin-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` - -## Testing -You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): -```bash -airbyte-ci connectors --name=source-linkedin-pages test -``` - -### Customizing acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-linkedin-pages test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). -3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/linkedin-pages.md`). -5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). -6. Pat yourself on the back for being an awesome contributor. -7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - diff --git a/source-linkedin-pages/acceptance-test-config.yml b/source-linkedin-pages/acceptance-test-config.yml deleted file mode 100644 index 535ba5a472..0000000000 --- a/source-linkedin-pages/acceptance-test-config.yml +++ /dev/null @@ -1,29 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-linkedin-pages:dev -test_strictness_level: low -acceptance_tests: - spec: - tests: - - spec_path: "source_linkedin_pages/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.0" - connection: - tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.0" - basic_read: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - full_refresh: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/source-linkedin-pages/bootstrap.md b/source-linkedin-pages/bootstrap.md deleted file mode 100644 index 90e5cbd5eb..0000000000 --- a/source-linkedin-pages/bootstrap.md +++ /dev/null @@ -1,7 +0,0 @@ -The LinkedIn Marketing Developer Platform API can be used to pull data from LinkedIn Organizations such as company page details, follower counts, follower statistics, and all sorts of organic content data. - -You must have a LinkedIn Developers' App created in order to request access to the Marketing Developer Platform API. API Access approval takes up to 72 hours after submitting a ~15-question form. - -The app also must be verified by an admin of the LinkedIn organization your app is created for. Once the app is "verified" and granted access to the Marketing Developer Platform API, you can use their easy-peasy OAuth Token Tools to generate access tokens **and** refresh tokens. - -You can access the `client id` and `client secret` in the **Auth** tab of the app dashboard to round out all of the authorization needs you may have. \ No newline at end of file diff --git a/source-linkedin-pages/icon.svg b/source-linkedin-pages/icon.svg deleted file mode 100644 index 246cc94447..0000000000 --- a/source-linkedin-pages/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/source-linkedin-pages/integration_tests/__init__.py b/source-linkedin-pages/integration_tests/__init__.py deleted file mode 100644 index 46b7376756..0000000000 --- a/source-linkedin-pages/integration_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/source-linkedin-pages/integration_tests/abnormal_state.json b/source-linkedin-pages/integration_tests/abnormal_state.json deleted file mode 100644 index 8f9f142bd9..0000000000 --- a/source-linkedin-pages/integration_tests/abnormal_state.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "organization_lookup": { - "lastModified": "2050-01-01" - }, - "follower_statistics": { - "lastModified": "2050-01-01" - }, - "page_statistics": { - "lastModified": "2050-01-01" - }, - "share_statistics": { - "lastModified": "2050-01-01" - }, - "shares": { - "lastModified": "2050-01-01" - }, - "total_follower_count": { - "end_date": "2050-01-01" - }, - "ugc_posts": { - "end_date": "2050-01-01" - } -} diff --git a/source-linkedin-pages/integration_tests/acceptance.py b/source-linkedin-pages/integration_tests/acceptance.py deleted file mode 100644 index d49b558823..0000000000 --- a/source-linkedin-pages/integration_tests/acceptance.py +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pytest - -pytest_plugins = ("connector_acceptance_test.plugin",) - - -@pytest.fixture(scope="session", autouse=True) -def connector_setup(): - yield diff --git a/source-linkedin-pages/integration_tests/configured_catalog.json b/source-linkedin-pages/integration_tests/configured_catalog.json deleted file mode 100644 index c0e203e604..0000000000 --- a/source-linkedin-pages/integration_tests/configured_catalog.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "organization_lookup", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "follower_statistics", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "share_statistics", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "total_follower_count", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/source-linkedin-pages/integration_tests/invalid_config.json b/source-linkedin-pages/integration_tests/invalid_config.json deleted file mode 100644 index a0fa475e00..0000000000 --- a/source-linkedin-pages/integration_tests/invalid_config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "org_id": "12345678", - "credentials": { - "auth_method": "access_token", - "access_token": "wrong_token_sra6ibiw0ZWEdMnC0ZizeD1gLRQP6u1pkQl" - } -} diff --git a/source-linkedin-pages/integration_tests/sample_config.json b/source-linkedin-pages/integration_tests/sample_config.json deleted file mode 100644 index 7399ec4e79..0000000000 --- a/source-linkedin-pages/integration_tests/sample_config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "org_id": "12345678", - "credentials": { - "auth_method": "access_token", - "access_token": "example_token_string123" - } -} diff --git a/source-linkedin-pages/integration_tests/sample_state.json b/source-linkedin-pages/integration_tests/sample_state.json deleted file mode 100644 index 3587e57982..0000000000 --- a/source-linkedin-pages/integration_tests/sample_state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "todo-stream-name": { - "todo-field-name": "value" - } -} diff --git a/source-linkedin-pages/main.py b/source-linkedin-pages/main.py deleted file mode 100644 index a6068720a6..0000000000 --- a/source-linkedin-pages/main.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from source_linkedin_pages.run import run - -if __name__ == "__main__": - run() diff --git a/source-linkedin-pages/metadata.yaml b/source-linkedin-pages/metadata.yaml deleted file mode 100644 index 3d2772011d..0000000000 --- a/source-linkedin-pages/metadata.yaml +++ /dev/null @@ -1,29 +0,0 @@ -data: - ab_internal: - ql: 200 - sl: 100 - connectorSubtype: api - connectorType: source - definitionId: af54297c-e8f8-4d63-a00d-a94695acc9d3 - dockerImageTag: 1.0.2 - dockerRepository: airbyte/source-linkedin-pages - documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-pages - githubIssueLabel: source-linkedin-pages - icon: linkedin.svg - license: MIT - name: LinkedIn Pages - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-linkedin-pages - registries: - cloud: - enabled: true - oss: - enabled: true - releaseStage: alpha - supportLevel: community - tags: - - language:python - - cdk:python -metadataSpecVersion: "1.0" diff --git a/source-linkedin-pages/setup.py b/source-linkedin-pages/setup.py deleted file mode 100644 index 1b491a1f3e..0000000000 --- a/source-linkedin-pages/setup.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", - "pendulum~=2.1", -] - -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", -] - -setup( - entry_points={ - "console_scripts": [ - "source-linkedin-pages=source_linkedin_pages.run:run", - ], - }, - name="source_linkedin_pages", - description="Source implementation for Linkedin Company Pages.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={ - "": [ - # Include yaml files in the package (if any) - "*.yml", - "*.yaml", - # Include all json files in the package, up to 4 levels deep - "*.json", - "*/*.json", - "*/*/*.json", - "*/*/*/*.json", - "*/*/*/*/*.json", - ] - }, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/source-linkedin-pages/source_linkedin_pages/__main__.py b/source-linkedin-pages/source_linkedin_pages/__main__.py new file mode 100644 index 0000000000..a0684e25e7 --- /dev/null +++ b/source-linkedin-pages/source_linkedin_pages/__main__.py @@ -0,0 +1,13 @@ +import estuary_cdk.pydantic_polyfill # Must be first. # noqa + +import asyncio +from estuary_cdk import shim_airbyte_cdk +from source_linkedin_pages import SourceLinkedinPages + +asyncio.run( + shim_airbyte_cdk.CaptureShim( + delegate=SourceLinkedinPages(), + oauth2=None, + schema_inference=True, + ).serve() +) diff --git a/source-linkedin-pages/source_linkedin_pages/run.py b/source-linkedin-pages/source_linkedin_pages/run.py deleted file mode 100644 index 4f0787573f..0000000000 --- a/source-linkedin-pages/source_linkedin_pages/run.py +++ /dev/null @@ -1,14 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import sys - -from airbyte_cdk.entrypoint import launch -from source_linkedin_pages import SourceLinkedinPages - - -def run(): - source = SourceLinkedinPages() - launch(source, sys.argv[1:]) diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index f29db87853..884277945d 100644 --- a/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -4,56 +4,101 @@ "additionalProperties": true, "properties": { "followerCountsByStaffCountRange": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true } }, "followerCountsByFunction": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true } }, "followerCountsByAssociationType": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true } }, "followerCountsBySeniority": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true } }, "followerCountsByRegion": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true } }, "organizationalEntity": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "followerCountsByCountry": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true } }, "followerCountsByIndustry": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true } } } -} +} \ No newline at end of file diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index 34d3275870..cd3d37e33a 100644 --- a/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -1,278 +1,497 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "additionalProperties": true, - "type": ["null", "object"], + "type": "object", "properties": { "vanityName": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "localizedName": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "website": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } }, "foundedOn": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "year": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } }, "groups": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { "items": {} } }, "description": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } }, "versionTag": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "coverPhotoV2": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "cropped": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "original": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "cropInfo": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "x": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "width": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "y": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "height": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } } } }, "defaultLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "organizationType": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "alternativeNames": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { "items": {} } }, "specialties": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { "items": {} } }, "staffCountRange": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "localizedSpecialties": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { "items": {} } }, "industries": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } }, "name": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } }, "primaryOrganizationType": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "locations": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "locationType": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "description": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "localized": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "en_US": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "preferredLocale": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "language": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } }, "address": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "geographicArea": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "country": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "city": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "line1": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "postalCode": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "localizedDescription": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "geoLocation": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "streetAddressFieldState": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } }, "id": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "localizedDescription": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "$URN": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "localizedWebsite": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "logoV2": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "cropped": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "original": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "cropInfo": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "x": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "width": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "y": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "height": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } } } } } -} +} \ No newline at end of file diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json index 77129bc9d5..b88202f7ae 100644 --- a/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json +++ b/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -4,37 +4,72 @@ "additionalProperties": true, "properties": { "totalShareStatistics": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "uniqueImpressionsCount": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "clickCount": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "engagement": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] }, "likeCount": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "commentCount": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "shareCount": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "commentMentionsCount": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "impressionCount": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, "shareMentionsCount": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } }, - "organizationalEntity": { "type": ["null", "string"] } + "organizationalEntity": { + "type": [ + "null", + "string" + ] + } } -} +} \ No newline at end of file diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json b/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json index 84387c6c37..9458aa1088 100644 --- a/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json +++ b/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json @@ -1,10 +1,13 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "additionalProperties": true, - "type": ["null", "object"], + "type": "object", "properties": { "firstDegreeSize": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] } } -} +} \ No newline at end of file diff --git a/source-linkedin-pages/source_linkedin_pages/source.py b/source-linkedin-pages/source_linkedin_pages/source.py index 93b7ef3f75..8c6c50ac3f 100644 --- a/source-linkedin-pages/source_linkedin_pages/source.py +++ b/source-linkedin-pages/source_linkedin_pages/source.py @@ -4,19 +4,29 @@ from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple - +from typing import ( + Any, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Tuple, +) +import urllib.parse import requests from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator +from airbyte_cdk.sources.streams.http.auth import ( + Oauth2Authenticator, + TokenAuthenticator, +) class LinkedinPagesStream(HttpStream, ABC): - url_base = "https://api.linkedin.com/v2/" primary_key = None @@ -33,11 +43,16 @@ def path(self, **kwargs) -> str: """Returns the API endpoint path for stream, from `endpoint` class attribute.""" return self.endpoint - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + def next_page_token( + self, response: requests.Response + ) -> Optional[Mapping[str, Any]]: return None def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, ) -> Iterable[Mapping]: return [response.json()] @@ -55,40 +70,67 @@ def should_retry(self, response: requests.Response) -> bool: class OrganizationLookup(LinkedinPagesStream): - def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - + def path( + self, stream_state: Mapping[str, Any], **kwargs + ) -> MutableMapping[str, Any]: path = f"organizations/{self.org}" return path class FollowerStatistics(LinkedinPagesStream): - def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - - path = f"organizationalEntityFollowerStatistics?q=organizationalEntity&organizationalEntity=urn:li:organization:{self.org}" + def path( + self, stream_state: Mapping[str, Any], **kwargs + ) -> MutableMapping[str, Any]: + base_path = "organizationalEntityFollowerStatistics?" + path = base_path + urllib.parse.urlencode( + { + "q": "organizationalEntity", + "organizationalEntity": f"urn:li:organization:{self.org}", + } + ) return path def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, ) -> Iterable[Mapping]: yield from response.json().get("elements") class ShareStatistics(LinkedinPagesStream): - def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - - path = f"organizationalEntityShareStatistics?q=organizationalEntity&organizationalEntity=urn%3Ali%3Aorganization%3A{self.org}" + def path( + self, stream_state: Mapping[str, Any], **kwargs + ) -> MutableMapping[str, Any]: + base_path = "organizationalEntityShareStatistics?" + path = base_path + urllib.parse.urlencode( + { + "q": "organizationalEntity", + "organizationalEntity": f"urn:li:organization:{self.org}", + } + ) return path def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, ) -> Iterable[Mapping]: yield from response.json().get("elements") class TotalFollowerCount(LinkedinPagesStream): - def path(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - - path = f"networkSizes/urn:li:organization:{self.org}?edgeType=CompanyFollowedByMember" + def path( + self, stream_state: Mapping[str, Any], **kwargs + ) -> MutableMapping[str, Any]: + base_path = f"networkSizes/urn:li:organization:{self.org}?" + path = base_path + urllib.parse.urlencode( + { + "edgeType": "CompanyFollowedByMember", + } + ) return path @@ -111,7 +153,11 @@ def get_authenticator(cls, config: Mapping[str, Any]) -> TokenAuthenticator: auth_method = config.get("credentials", {}).get("auth_method") if not auth_method or auth_method == "access_token": # support of backward compatibility with old exists configs - access_token = config["credentials"]["access_token"] if auth_method else config["access_token"] + access_token = ( + config["credentials"]["access_token"] + if auth_method + else config["access_token"] + ) return TokenAuthenticator(token=access_token) elif auth_method == "oAuth2.0": return Oauth2Authenticator( @@ -122,7 +168,9 @@ def get_authenticator(cls, config: Mapping[str, Any]) -> TokenAuthenticator: ) raise Exception("incorrect input parameters") - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: + def check_connection( + self, logger: AirbyteLogger, config: Mapping[str, Any] + ) -> Tuple[bool, any]: # RUN $ python main.py check --config secrets/config.json """ diff --git a/source-linkedin-pages/source_linkedin_pages/utils.py b/source-linkedin-pages/source_linkedin_pages/utils.py index 3f98d82156..ac63f6e0dc 100644 --- a/source-linkedin-pages/source_linkedin_pages/utils.py +++ b/source-linkedin-pages/source_linkedin_pages/utils.py @@ -30,9 +30,11 @@ def get_parent_stream_values(record: Dict, key_value_map: Dict) -> Dict: def transform_change_audit_stamps( - record: Dict, dict_key: str = "changeAuditStamps", props: List = ["created", "lastModified"], fields: List = ["time"] + record: Dict, + dict_key: str = "changeAuditStamps", + props: List = ["created", "lastModified"], + fields: List = ["time"], ) -> Mapping[str, Any]: - """ :: EXAMPLE `changeAuditStamps` input structure: { @@ -53,7 +55,9 @@ def transform_change_audit_stamps( for prop in props: # Update dict with flatten key:value for field in fields: - record[prop] = pdm.from_timestamp(target_dict.get(prop).get(field) / 1000).to_datetime_string() + record[prop] = pdm.from_timestamp( + target_dict.get(prop).get(field) / 1000 + ).to_datetime_string() record.pop(dict_key) return record @@ -90,7 +94,6 @@ def transform_date_range( props: List = ["start", "end"], fields: List = ["year", "month", "day"], ) -> Mapping[str, Any]: - """ :: EXAMPLE `dateRange` input structure in Analytics streams: { @@ -106,7 +109,17 @@ def transform_date_range( } """ # define list of tmp keys for cleanup. - keys_to_remove = [dict_key, "start.day", "start.month", "start.year", "end.day", "end.month", "end.year", "start", "end"] + keys_to_remove = [ + dict_key, + "start.day", + "start.month", + "start.year", + "end.day", + "end.month", + "end.year", + "start", + "end", + ] target_dict: Dict = record.get(dict_key) for prop in props: @@ -114,7 +127,12 @@ def transform_date_range( for field in fields: record.update(**{f"{prop}.{field}": target_dict.get(prop).get(field)}) # We build `start_date` & `end_date` fields from nested structure. - record.update(**{"start_date": date_str_from_date_range(record, "start"), "end_date": date_str_from_date_range(record, "end")}) + record.update( + **{ + "start_date": date_str_from_date_range(record, "start"), + "end_date": date_str_from_date_range(record, "end"), + } + ) # Cleanup tmp fields & nested used parts for key in keys_to_remove: if key in record: @@ -126,7 +144,6 @@ def transform_targeting_criteria( record: Dict, dict_key: str = "targetingCriteria", ) -> Mapping[str, Any]: - """ :: EXAMPLE `targetingCriteria` input structure: { @@ -262,7 +279,6 @@ def transform_variables( record: Dict, dict_key: str = "variables", ) -> Mapping[str, Any]: - """ :: EXAMPLE `variables` input: { @@ -296,7 +312,9 @@ def transform_variables( record["variables"]["values"] = [] for key, value in params.items(): # convert various datatypes of values into the string - record["variables"]["values"].append({"key": key, "value": json.dumps(value, ensure_ascii=True)}) + record["variables"]["values"].append( + {"key": key, "value": json.dumps(value, ensure_ascii=True)} + ) # Clean the nested structure record["variables"].pop("data") return record @@ -308,7 +326,6 @@ def transform_data(records: List) -> Iterable[Mapping]: to be properly normalised in the destination. """ for record in records: - if "changeAuditStamps" in record: record = transform_change_audit_stamps(record) diff --git a/source-linkedin-pages/unit_tests/__init__.py b/source-linkedin-pages/unit_tests/__init__.py deleted file mode 100644 index 46b7376756..0000000000 --- a/source-linkedin-pages/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# From 028473219b6ce467f0aeb1183598ba25f0ba4640 Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 28 Mar 2024 15:19:31 -0300 Subject: [PATCH 68/96] source-linkedin-pages: Add tests for capturing, discovering, and specifying tests: Add test capture files and credentials --- source-linkedin-pages/acmeCo/flow.yaml | 34 + .../acmeCo/follower_statistics.schema.yaml | 80 ++ .../acmeCo/organization_lookup.schema.yaml | 333 +++++ .../acmeCo/share_statistics.schema.yaml | 58 + source-linkedin-pages/connector_config.yaml | 20 + source-linkedin-pages/test.flow.yaml | 29 + source-linkedin-pages/tests/__init__.py | 0 .../tests/snapshots/capture.stdout.json | 1192 +++++++++++++++++ .../tests/snapshots/discover.stdout.json | 779 +++++++++++ .../tests/snapshots/spec.stdout.json | 124 ++ source-linkedin-pages/tests/test_snapshots.py | 86 ++ 11 files changed, 2735 insertions(+) create mode 100644 source-linkedin-pages/acmeCo/flow.yaml create mode 100644 source-linkedin-pages/acmeCo/follower_statistics.schema.yaml create mode 100644 source-linkedin-pages/acmeCo/organization_lookup.schema.yaml create mode 100644 source-linkedin-pages/acmeCo/share_statistics.schema.yaml create mode 100644 source-linkedin-pages/connector_config.yaml create mode 100644 source-linkedin-pages/test.flow.yaml create mode 100644 source-linkedin-pages/tests/__init__.py create mode 100644 source-linkedin-pages/tests/snapshots/capture.stdout.json create mode 100644 source-linkedin-pages/tests/snapshots/discover.stdout.json create mode 100644 source-linkedin-pages/tests/snapshots/spec.stdout.json create mode 100644 source-linkedin-pages/tests/test_snapshots.py diff --git a/source-linkedin-pages/acmeCo/flow.yaml b/source-linkedin-pages/acmeCo/flow.yaml new file mode 100644 index 0000000000..24fdc4def4 --- /dev/null +++ b/source-linkedin-pages/acmeCo/flow.yaml @@ -0,0 +1,34 @@ +--- +collections: + acmeCo/follower_statistics: + schema: follower_statistics.schema.yaml + key: + - /_meta/row_id + acmeCo/organization_lookup: + schema: organization_lookup.schema.yaml + key: + - /_meta/row_id + acmeCo/share_statistics: + schema: share_statistics.schema.yaml + key: + - /_meta/row_id + acmeCo/total_follower_count: + schema: + $schema: "http://json-schema.org/draft-04/schema#" + additionalProperties: true + type: object + properties: + firstDegreeSize: + type: + - "null" + - integer + _meta: + type: object + properties: + row_id: + type: integer + required: + - row_id + x-infer-schema: true + key: + - /_meta/row_id diff --git a/source-linkedin-pages/acmeCo/follower_statistics.schema.yaml b/source-linkedin-pages/acmeCo/follower_statistics.schema.yaml new file mode 100644 index 0000000000..696064c670 --- /dev/null +++ b/source-linkedin-pages/acmeCo/follower_statistics.schema.yaml @@ -0,0 +1,80 @@ +--- +$schema: "http://json-schema.org/draft-04/schema#" +type: object +additionalProperties: true +properties: + followerCountsByStaffCountRange: + type: + - "null" + - array + items: + type: + - "null" + - object + additionalProperties: true + followerCountsByFunction: + type: + - "null" + - array + items: + type: + - "null" + - object + additionalProperties: true + followerCountsByAssociationType: + type: + - "null" + - array + items: + type: + - "null" + - object + additionalProperties: true + followerCountsBySeniority: + type: + - "null" + - array + items: + type: + - "null" + - object + additionalProperties: true + followerCountsByRegion: + type: + - "null" + - array + items: + type: + - "null" + - object + additionalProperties: true + organizationalEntity: + type: + - "null" + - string + followerCountsByCountry: + type: + - "null" + - array + items: + type: + - "null" + - object + additionalProperties: true + followerCountsByIndustry: + type: + - "null" + - array + items: + type: + - "null" + - object + additionalProperties: true + _meta: + type: object + properties: + row_id: + type: integer + required: + - row_id +x-infer-schema: true diff --git a/source-linkedin-pages/acmeCo/organization_lookup.schema.yaml b/source-linkedin-pages/acmeCo/organization_lookup.schema.yaml new file mode 100644 index 0000000000..75766bd6fd --- /dev/null +++ b/source-linkedin-pages/acmeCo/organization_lookup.schema.yaml @@ -0,0 +1,333 @@ +--- +$schema: "http://json-schema.org/draft-04/schema#" +additionalProperties: true +type: object +properties: + vanityName: + type: + - "null" + - string + localizedName: + type: + - "null" + - string + website: + type: + - "null" + - object + properties: + localized: + type: + - "null" + - object + properties: + en_US: + type: + - "null" + - string + preferredLocale: + type: + - "null" + - object + properties: + country: + type: + - "null" + - string + language: + type: + - "null" + - string + foundedOn: + type: + - "null" + - object + properties: + year: + type: + - "null" + - integer + groups: + type: + - "null" + - array + items: + items: {} + description: + type: + - "null" + - object + properties: + localized: + type: + - "null" + - object + properties: + en_US: + type: + - "null" + - string + preferredLocale: + type: + - "null" + - object + properties: + country: + type: + - "null" + - string + language: + type: + - "null" + - string + versionTag: + type: + - "null" + - string + coverPhotoV2: + type: + - "null" + - object + properties: + cropped: + type: + - "null" + - string + original: + type: + - "null" + - string + cropInfo: + type: + - "null" + - object + properties: + x: + type: + - "null" + - integer + width: + type: + - "null" + - integer + y: + type: + - "null" + - integer + height: + type: + - "null" + - integer + defaultLocale: + type: + - "null" + - object + properties: + country: + type: + - "null" + - string + language: + type: + - "null" + - string + organizationType: + type: + - "null" + - string + alternativeNames: + type: + - "null" + - array + items: + items: {} + specialties: + type: + - "null" + - array + items: + items: {} + staffCountRange: + type: + - "null" + - string + localizedSpecialties: + type: + - "null" + - array + items: + items: {} + industries: + type: + - "null" + - array + items: + type: + - "null" + - string + name: + type: + - "null" + - object + properties: + localized: + type: + - "null" + - object + properties: + en_US: + type: + - "null" + - string + preferredLocale: + type: + - "null" + - object + properties: + country: + type: + - "null" + - string + language: + type: + - "null" + - string + primaryOrganizationType: + type: + - "null" + - string + locations: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + locationType: + type: + - "null" + - string + description: + type: + - "null" + - object + properties: + localized: + type: + - "null" + - object + properties: + en_US: + type: + - "null" + - string + preferredLocale: + type: + - "null" + - object + properties: + country: + type: + - "null" + - string + language: + type: + - "null" + - string + address: + type: + - "null" + - object + properties: + geographicArea: + type: + - "null" + - string + country: + type: + - "null" + - string + city: + type: + - "null" + - string + line1: + type: + - "null" + - string + postalCode: + type: + - "null" + - string + localizedDescription: + type: + - "null" + - string + geoLocation: + type: + - "null" + - string + streetAddressFieldState: + type: + - "null" + - string + id: + type: + - "null" + - integer + localizedDescription: + type: + - "null" + - string + $URN: + type: + - "null" + - string + localizedWebsite: + type: + - "null" + - string + logoV2: + type: + - "null" + - object + properties: + cropped: + type: + - "null" + - string + original: + type: + - "null" + - string + cropInfo: + type: + - "null" + - object + properties: + x: + type: + - "null" + - integer + width: + type: + - "null" + - integer + y: + type: + - "null" + - integer + height: + type: + - "null" + - integer + _meta: + type: object + properties: + row_id: + type: integer + required: + - row_id +x-infer-schema: true diff --git a/source-linkedin-pages/acmeCo/share_statistics.schema.yaml b/source-linkedin-pages/acmeCo/share_statistics.schema.yaml new file mode 100644 index 0000000000..1ad52b5769 --- /dev/null +++ b/source-linkedin-pages/acmeCo/share_statistics.schema.yaml @@ -0,0 +1,58 @@ +--- +$schema: "http://json-schema.org/draft-04/schema#" +type: object +additionalProperties: true +properties: + totalShareStatistics: + type: + - "null" + - object + properties: + uniqueImpressionsCount: + type: + - "null" + - integer + clickCount: + type: + - "null" + - integer + engagement: + type: + - "null" + - number + likeCount: + type: + - "null" + - integer + commentCount: + type: + - "null" + - integer + shareCount: + type: + - "null" + - integer + commentMentionsCount: + type: + - "null" + - integer + impressionCount: + type: + - "null" + - integer + shareMentionsCount: + type: + - "null" + - integer + organizationalEntity: + type: + - "null" + - string + _meta: + type: object + properties: + row_id: + type: integer + required: + - row_id +x-infer-schema: true diff --git a/source-linkedin-pages/connector_config.yaml b/source-linkedin-pages/connector_config.yaml new file mode 100644 index 0000000000..269e09452b --- /dev/null +++ b/source-linkedin-pages/connector_config.yaml @@ -0,0 +1,20 @@ +credentials: + auth_method: oAuth2.0 + client_id: 782dm89v4dijz4 + client_secret_sops: ENC[AES256_GCM,data:NgPttBowFqzdyAuZlRl5Fg==,iv:7By8FHimHGpYXCA57vqTiFH++uVLFaqo/ymMK+H7+mE=,tag:naHF50oMbPsduEr1jiXkRA==,type:str] + refresh_token: AQXMroM7SCExt3p0WRkcgGdZawBk9eZgIOao84vEB6W7d--73g-XyjZMdAqbbc0hwZIThulSW0ZsSTykHm85vvlO1TOlFzN6IG_Zaa6AjhRzVSe9Eo2YFG15Ip7keUH8htHlWfGtVc67C2OlNH9UFB_zNR56FPKf8nDEZCGPN65G_CP2NxTXxiqAY8rzhD4L69Xo-HV7bwuRLuM6JY20o9HAVmzgyjbUorvv2w_jwSN5IW_KTiqiw0jWjXsldIM9KTQpmIEHfC6hzFkxe39dVr6XSWfYNRNMUg85v3YJ3imwXcwWkQARbTPNOyFBSJg4jRBsVbaIVmVFDO8AXHe2TRv9Yc8J2Q +org_id: "65266256" +sops: + kms: [] + gcp_kms: + - resource_id: projects/estuary-theatre/locations/us-central1/keyRings/connector-keyring/cryptoKeys/connector-repository + created_at: "2024-01-03T20:18:13Z" + enc: CiQAdmEdwg//2BiM2VRfoml0CiaFrzldf9Rw0F0k1eVzXUyOqZkSSQCVvC1z1UacMnOFeY14RV/T1kf6C1XsJ3GslDgpq4Is8fo/cfg9STPjfPEq7ZYgWzMwMvWgZUOJHI0BagsAYEcRo0noe4aW5GM= + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2024-03-15T16:40:36Z" + mac: ENC[AES256_GCM,data:wTwsj5Ku2XxwPMNFg1/G+Tz83SD8hXIEBIUu227qKJvaJl9vcTXz9Vtvw2AQmhYV7hrf3TW+3nz7i0v0Z5F0kpvqzBXeYF+YcCraJmtWmWxlCJdZrMmyvzH9RQd4+Ys71D0JMDt7HV3rpMPiCW4WgQFiSldmw9kFSZ2JCxI3URI=,iv:owKyAuGA45Gs9RwDQKmKv3ykILZAkZ5qXE/b9gP2BKw=,tag:S69uEmvd7/2K6RdwIYc4og==,type:str] + pgp: [] + encrypted_suffix: _sops + version: 3.8.1 diff --git a/source-linkedin-pages/test.flow.yaml b/source-linkedin-pages/test.flow.yaml new file mode 100644 index 0000000000..6ba19fdaec --- /dev/null +++ b/source-linkedin-pages/test.flow.yaml @@ -0,0 +1,29 @@ +--- +import: + - acmeCo/flow.yaml +captures: + acmeCo/source-linkedin-pages: + endpoint: + local: + command: + - python + - "-m" + - source_linkedin_pages + config: connector_config.yaml + bindings: + - resource: + stream: organization_lookup + syncMode: full_refresh + target: acmeCo/organization_lookup + - resource: + stream: follower_statistics + syncMode: full_refresh + target: acmeCo/follower_statistics + - resource: + stream: share_statistics + syncMode: full_refresh + target: acmeCo/share_statistics + - resource: + stream: total_follower_count + syncMode: full_refresh + target: acmeCo/total_follower_count diff --git a/source-linkedin-pages/tests/__init__.py b/source-linkedin-pages/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source-linkedin-pages/tests/snapshots/capture.stdout.json b/source-linkedin-pages/tests/snapshots/capture.stdout.json new file mode 100644 index 0000000000..9423b5509e --- /dev/null +++ b/source-linkedin-pages/tests/snapshots/capture.stdout.json @@ -0,0 +1,1192 @@ +[ + [ + "acmeCo/organization_lookup", + { + "$URN": "urn:li:organization:65266256", + "_meta": { + "op": "u", + "row_id": 0 + }, + "alternativeNames": [], + "coverPhotoV2": { + "cropInfo": { + "height": 212, + "width": 1256, + "x": 0, + "y": 97 + }, + "cropped": "urn:li:digitalmediaAsset:C561BAQGB95zZDFql-Q", + "original": "urn:li:digitalmediaAsset:C561BAQGB95zZDFql-Q" + }, + "defaultLocale": { + "country": "US", + "language": "en" + }, + "description": { + "localized": { + "en_US": "Estuary helps organizations gain real-time access to their data without having to manage infrastructure. Capture data from SaaS or technology sources, transform it and materialize it back into the same types of systems all with millisecond latency." + }, + "preferredLocale": { + "country": "US", + "language": "en" + } + }, + "foundedOn": { + "year": 2019 + }, + "groups": [], + "id": 65266256, + "industries": [ + "urn:li:industry:4" + ], + "localizedDescription": "Estuary helps organizations gain real-time access to their data without having to manage infrastructure. Capture data from SaaS or technology sources, transform it and materialize it back into the same types of systems all with millisecond latency.", + "localizedName": "Estuary", + "localizedSpecialties": [], + "localizedWebsite": "http://estuary.dev", + "locations": [ + { + "address": { + "city": "New York", + "country": "US", + "geographicArea": "NY", + "line1": " 244 Fifth Avenue", + "line2": "Suite 1277", + "postalCode": "10001" + }, + "description": { + "localized": { + "en_US": "Headquarters" + }, + "preferredLocale": { + "country": "US", + "language": "en" + } + }, + "geoLocation": "urn:li:geo:104207524", + "localizedDescription": "Headquarters", + "locationType": "HEADQUARTERS", + "streetAddressFieldState": "UNSET_OPT_OUT" + }, + { + "address": { + "city": "Columbus", + "country": "US", + "geographicArea": "Ohio", + "line1": "West State St", + "postalCode": "43215" + }, + "geoLocation": "urn:li:geo:100278071", + "locationType": "OTHER", + "streetAddressFieldState": "UNSET_OPT_OUT" + } + ], + "logoV2": { + "cropInfo": { + "height": 0, + "width": 0, + "x": 0, + "y": 0 + }, + "cropped": "urn:li:digitalmediaAsset:C560BAQGNlRBEM6qhzQ", + "original": "urn:li:digitalmediaAsset:C560BAQGNlRBEM6qhzQ" + }, + "name": { + "localized": { + "en_US": "Estuary" + }, + "preferredLocale": { + "country": "US", + "language": "en" + } + }, + "organizationType": "PRIVATELY_HELD", + "primaryOrganizationType": "NONE", + "specialties": [], + "staffCountRange": "SIZE_11_TO_50", + "vanityName": "estuary-tech", + "versionTag": "4106875789", + "website": { + "localized": { + "en_US": "http://estuary.dev" + }, + "preferredLocale": { + "country": "US", + "language": "en" + } + } + } + ], + [ + "acmeCo/follower_statistics", + { + "_meta": { + "op": "u", + "row_id": 0 + }, + "followerCountsByAssociationType": [ + { + "associationType": "EMPLOYEE", + "followerCounts": { + "organicFollowerCount": 24, + "paidFollowerCount": 0 + } + } + ], + "followerCountsByFunction": [ + { + "followerCounts": { + "organicFollowerCount": 1973, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:8" + }, + { + "followerCounts": { + "organicFollowerCount": 983, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:13" + }, + { + "followerCounts": { + "organicFollowerCount": 395, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:4" + }, + { + "followerCounts": { + "organicFollowerCount": 185, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:25" + }, + { + "followerCounts": { + "organicFollowerCount": 154, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:16" + }, + { + "followerCounts": { + "organicFollowerCount": 151, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:7" + }, + { + "followerCounts": { + "organicFollowerCount": 143, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:18" + }, + { + "followerCounts": { + "organicFollowerCount": 130, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:15" + }, + { + "followerCounts": { + "organicFollowerCount": 129, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:24" + }, + { + "followerCounts": { + "organicFollowerCount": 114, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:19" + }, + { + "followerCounts": { + "organicFollowerCount": 88, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:3" + }, + { + "followerCounts": { + "organicFollowerCount": 62, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:6" + }, + { + "followerCounts": { + "organicFollowerCount": 54, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:20" + }, + { + "followerCounts": { + "organicFollowerCount": 51, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:10" + }, + { + "followerCounts": { + "organicFollowerCount": 50, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:26" + }, + { + "followerCounts": { + "organicFollowerCount": 46, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:2" + }, + { + "followerCounts": { + "organicFollowerCount": 39, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:9" + }, + { + "followerCounts": { + "organicFollowerCount": 33, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:5" + }, + { + "followerCounts": { + "organicFollowerCount": 33, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:12" + }, + { + "followerCounts": { + "organicFollowerCount": 27, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:22" + }, + { + "followerCounts": { + "organicFollowerCount": 15, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:1" + }, + { + "followerCounts": { + "organicFollowerCount": 13, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:17" + }, + { + "followerCounts": { + "organicFollowerCount": 10, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:14" + }, + { + "followerCounts": { + "organicFollowerCount": 9, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:11" + }, + { + "followerCounts": { + "organicFollowerCount": 5, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:23" + }, + { + "followerCounts": { + "organicFollowerCount": 1, + "paidFollowerCount": 0 + }, + "function": "urn:li:function:21" + } + ], + "followerCountsByIndustry": [ + { + "followerCounts": { + "organicFollowerCount": 1318, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:4" + }, + { + "followerCounts": { + "organicFollowerCount": 1101, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:96" + }, + { + "followerCounts": { + "organicFollowerCount": 365, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:6" + }, + { + "followerCounts": { + "organicFollowerCount": 216, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:80" + }, + { + "followerCounts": { + "organicFollowerCount": 170, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:68" + }, + { + "followerCounts": { + "organicFollowerCount": 167, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:43" + }, + { + "followerCounts": { + "organicFollowerCount": 126, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:11" + }, + { + "followerCounts": { + "organicFollowerCount": 87, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3132" + }, + { + "followerCounts": { + "organicFollowerCount": 80, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:2458" + }, + { + "followerCounts": { + "organicFollowerCount": 78, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3102" + }, + { + "followerCounts": { + "organicFollowerCount": 69, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1673" + }, + { + "followerCounts": { + "organicFollowerCount": 68, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:102" + }, + { + "followerCounts": { + "organicFollowerCount": 66, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:5" + }, + { + "followerCounts": { + "organicFollowerCount": 56, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:27" + }, + { + "followerCounts": { + "organicFollowerCount": 56, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:41" + }, + { + "followerCounts": { + "organicFollowerCount": 52, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:14" + }, + { + "followerCounts": { + "organicFollowerCount": 49, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:8" + }, + { + "followerCounts": { + "organicFollowerCount": 47, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3128" + }, + { + "followerCounts": { + "organicFollowerCount": 45, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:70" + }, + { + "followerCounts": { + "organicFollowerCount": 44, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:118" + }, + { + "followerCounts": { + "organicFollowerCount": 43, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:132" + }, + { + "followerCounts": { + "organicFollowerCount": 41, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:75" + }, + { + "followerCounts": { + "organicFollowerCount": 41, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:46" + }, + { + "followerCounts": { + "organicFollowerCount": 41, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:129" + }, + { + "followerCounts": { + "organicFollowerCount": 40, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:84" + }, + { + "followerCounts": { + "organicFollowerCount": 39, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:42" + }, + { + "followerCounts": { + "organicFollowerCount": 36, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1285" + }, + { + "followerCounts": { + "organicFollowerCount": 34, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:104" + }, + { + "followerCounts": { + "organicFollowerCount": 33, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1725" + }, + { + "followerCounts": { + "organicFollowerCount": 32, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1862" + }, + { + "followerCounts": { + "organicFollowerCount": 32, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:106" + }, + { + "followerCounts": { + "organicFollowerCount": 31, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:105" + }, + { + "followerCounts": { + "organicFollowerCount": 31, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:116" + }, + { + "followerCounts": { + "organicFollowerCount": 31, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:69" + }, + { + "followerCounts": { + "organicFollowerCount": 30, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:100" + }, + { + "followerCounts": { + "organicFollowerCount": 29, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3106" + }, + { + "followerCounts": { + "organicFollowerCount": 29, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:103" + }, + { + "followerCounts": { + "organicFollowerCount": 29, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:47" + }, + { + "followerCounts": { + "organicFollowerCount": 28, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:12" + }, + { + "followerCounts": { + "organicFollowerCount": 28, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1999" + }, + { + "followerCounts": { + "organicFollowerCount": 27, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:57" + }, + { + "followerCounts": { + "organicFollowerCount": 24, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1594" + }, + { + "followerCounts": { + "organicFollowerCount": 24, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:44" + }, + { + "followerCounts": { + "organicFollowerCount": 24, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:53" + }, + { + "followerCounts": { + "organicFollowerCount": 23, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:90" + }, + { + "followerCounts": { + "organicFollowerCount": 22, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:15" + }, + { + "followerCounts": { + "organicFollowerCount": 22, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:30" + }, + { + "followerCounts": { + "organicFollowerCount": 22, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:112" + }, + { + "followerCounts": { + "organicFollowerCount": 22, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:99" + }, + { + "followerCounts": { + "organicFollowerCount": 21, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3130" + }, + { + "followerCounts": { + "organicFollowerCount": 20, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:17" + }, + { + "followerCounts": { + "organicFollowerCount": 19, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:28" + }, + { + "followerCounts": { + "organicFollowerCount": 18, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:52" + }, + { + "followerCounts": { + "organicFollowerCount": 18, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:19" + }, + { + "followerCounts": { + "organicFollowerCount": 18, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1157" + }, + { + "followerCounts": { + "organicFollowerCount": 17, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1855" + }, + { + "followerCounts": { + "organicFollowerCount": 17, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1644" + }, + { + "followerCounts": { + "organicFollowerCount": 17, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:67" + }, + { + "followerCounts": { + "organicFollowerCount": 16, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:124" + }, + { + "followerCounts": { + "organicFollowerCount": 16, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:130" + }, + { + "followerCounts": { + "organicFollowerCount": 16, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:10" + }, + { + "followerCounts": { + "organicFollowerCount": 15, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:59" + }, + { + "followerCounts": { + "organicFollowerCount": 15, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:7" + }, + { + "followerCounts": { + "organicFollowerCount": 14, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:34" + }, + { + "followerCounts": { + "organicFollowerCount": 14, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:91" + }, + { + "followerCounts": { + "organicFollowerCount": 13, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1" + }, + { + "followerCounts": { + "organicFollowerCount": 13, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:24" + }, + { + "followerCounts": { + "organicFollowerCount": 13, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:97" + }, + { + "followerCounts": { + "organicFollowerCount": 13, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3107" + }, + { + "followerCounts": { + "organicFollowerCount": 12, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:2081" + }, + { + "followerCounts": { + "organicFollowerCount": 12, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:25" + }, + { + "followerCounts": { + "organicFollowerCount": 12, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:9" + }, + { + "followerCounts": { + "organicFollowerCount": 12, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:48" + }, + { + "followerCounts": { + "organicFollowerCount": 12, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:113" + }, + { + "followerCounts": { + "organicFollowerCount": 11, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:92" + }, + { + "followerCounts": { + "organicFollowerCount": 11, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:31" + }, + { + "followerCounts": { + "organicFollowerCount": 11, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:140" + }, + { + "followerCounts": { + "organicFollowerCount": 11, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:137" + }, + { + "followerCounts": { + "organicFollowerCount": 11, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3242" + }, + { + "followerCounts": { + "organicFollowerCount": 10, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:32" + }, + { + "followerCounts": { + "organicFollowerCount": 10, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:63" + }, + { + "followerCounts": { + "organicFollowerCount": 10, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3099" + }, + { + "followerCounts": { + "organicFollowerCount": 10, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:73" + }, + { + "followerCounts": { + "organicFollowerCount": 10, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1339" + }, + { + "followerCounts": { + "organicFollowerCount": 9, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:94" + }, + { + "followerCounts": { + "organicFollowerCount": 9, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:126" + }, + { + "followerCounts": { + "organicFollowerCount": 9, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1713" + }, + { + "followerCounts": { + "organicFollowerCount": 9, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:1925" + }, + { + "followerCounts": { + "organicFollowerCount": 9, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:82" + }, + { + "followerCounts": { + "organicFollowerCount": 9, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:147" + }, + { + "followerCounts": { + "organicFollowerCount": 9, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3241" + }, + { + "followerCounts": { + "organicFollowerCount": 9, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:87" + }, + { + "followerCounts": { + "organicFollowerCount": 9, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:36" + }, + { + "followerCounts": { + "organicFollowerCount": 8, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3103" + }, + { + "followerCounts": { + "organicFollowerCount": 8, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:13" + }, + { + "followerCounts": { + "organicFollowerCount": 8, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:121" + }, + { + "followerCounts": { + "organicFollowerCount": 8, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:139" + }, + { + "followerCounts": { + "organicFollowerCount": 8, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3131" + }, + { + "followerCounts": { + "organicFollowerCount": 8, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:109" + }, + { + "followerCounts": { + "organicFollowerCount": 8, + "paidFollowerCount": 0 + }, + "industry": "urn:li:industry:3127" + } + ], + "followerCountsBySeniority": [ + { + "followerCounts": { + "organicFollowerCount": 2620, + "paidFollowerCount": 0 + }, + "seniority": "urn:li:seniority:3" + }, + { + "followerCounts": { + "organicFollowerCount": 1417, + "paidFollowerCount": 0 + }, + "seniority": "urn:li:seniority:4" + }, + { + "followerCounts": { + "organicFollowerCount": 277, + "paidFollowerCount": 0 + }, + "seniority": "urn:li:seniority:6" + }, + { + "followerCounts": { + "organicFollowerCount": 146, + "paidFollowerCount": 0 + }, + "seniority": "urn:li:seniority:10" + }, + { + "followerCounts": { + "organicFollowerCount": 136, + "paidFollowerCount": 0 + }, + "seniority": "urn:li:seniority:2" + }, + { + "followerCounts": { + "organicFollowerCount": 136, + "paidFollowerCount": 0 + }, + "seniority": "urn:li:seniority:5" + }, + { + "followerCounts": { + "organicFollowerCount": 133, + "paidFollowerCount": 0 + }, + "seniority": "urn:li:seniority:7" + }, + { + "followerCounts": { + "organicFollowerCount": 98, + "paidFollowerCount": 0 + }, + "seniority": "urn:li:seniority:8" + }, + { + "followerCounts": { + "organicFollowerCount": 45, + "paidFollowerCount": 0 + }, + "seniority": "urn:li:seniority:9" + }, + { + "followerCounts": { + "organicFollowerCount": 6, + "paidFollowerCount": 0 + }, + "seniority": "urn:li:seniority:1" + } + ], + "followerCountsByStaffCountRange": [ + { + "followerCounts": { + "organicFollowerCount": 905, + "paidFollowerCount": 0 + }, + "staffCountRange": "SIZE_10001_OR_MORE" + }, + { + "followerCounts": { + "organicFollowerCount": 773, + "paidFollowerCount": 0 + }, + "staffCountRange": "SIZE_11_TO_50" + }, + { + "followerCounts": { + "organicFollowerCount": 725, + "paidFollowerCount": 0 + }, + "staffCountRange": "SIZE_51_TO_200" + }, + { + "followerCounts": { + "organicFollowerCount": 552, + "paidFollowerCount": 0 + }, + "staffCountRange": "SIZE_1001_TO_5000" + }, + { + "followerCounts": { + "organicFollowerCount": 513, + "paidFollowerCount": 0 + }, + "staffCountRange": "SIZE_2_TO_10" + }, + { + "followerCounts": { + "organicFollowerCount": 394, + "paidFollowerCount": 0 + }, + "staffCountRange": "SIZE_201_TO_500" + }, + { + "followerCounts": { + "organicFollowerCount": 306, + "paidFollowerCount": 0 + }, + "staffCountRange": "SIZE_501_TO_1000" + }, + { + "followerCounts": { + "organicFollowerCount": 223, + "paidFollowerCount": 0 + }, + "staffCountRange": "SIZE_5001_TO_10000" + }, + { + "followerCounts": { + "organicFollowerCount": 88, + "paidFollowerCount": 0 + }, + "staffCountRange": "SIZE_1" + } + ], + "organizationalEntity": "urn:li:organization:65266256" + } + ], + [ + "acmeCo/share_statistics", + { + "_meta": { + "op": "u", + "row_id": 0 + }, + "organizationalEntity": "urn:li:organization:65266256", + "totalShareStatistics": { + "clickCount": 1584, + "commentCount": 39, + "commentMentionsCount": 48, + "engagement": 0.042974090932691805, + "impressionCount": 57779, + "likeCount": 819, + "shareCount": 41, + "shareMentionsCount": 78, + "uniqueImpressionsCount": 21282 + } + } + ], + [ + "acmeCo/total_follower_count", + { + "_meta": { + "op": "u", + "row_id": 0 + }, + "firstDegreeSize": 6284 + } + ] +] \ No newline at end of file diff --git a/source-linkedin-pages/tests/snapshots/discover.stdout.json b/source-linkedin-pages/tests/snapshots/discover.stdout.json new file mode 100644 index 0000000000..6a14998a9d --- /dev/null +++ b/source-linkedin-pages/tests/snapshots/discover.stdout.json @@ -0,0 +1,779 @@ +[ + { + "recommendedName": "organization_lookup", + "resourceConfig": { + "stream": "organization_lookup", + "syncMode": "full_refresh" + }, + "documentSchema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "vanityName": { + "type": [ + "null", + "string" + ] + }, + "localizedName": { + "type": [ + "null", + "string" + ] + }, + "website": { + "type": [ + "null", + "object" + ], + "properties": { + "localized": { + "type": [ + "null", + "object" + ], + "properties": { + "en_US": { + "type": [ + "null", + "string" + ] + } + } + }, + "preferredLocale": { + "type": [ + "null", + "object" + ], + "properties": { + "country": { + "type": [ + "null", + "string" + ] + }, + "language": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "foundedOn": { + "type": [ + "null", + "object" + ], + "properties": { + "year": { + "type": [ + "null", + "integer" + ] + } + } + }, + "groups": { + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } + }, + "description": { + "type": [ + "null", + "object" + ], + "properties": { + "localized": { + "type": [ + "null", + "object" + ], + "properties": { + "en_US": { + "type": [ + "null", + "string" + ] + } + } + }, + "preferredLocale": { + "type": [ + "null", + "object" + ], + "properties": { + "country": { + "type": [ + "null", + "string" + ] + }, + "language": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "versionTag": { + "type": [ + "null", + "string" + ] + }, + "coverPhotoV2": { + "type": [ + "null", + "object" + ], + "properties": { + "cropped": { + "type": [ + "null", + "string" + ] + }, + "original": { + "type": [ + "null", + "string" + ] + }, + "cropInfo": { + "type": [ + "null", + "object" + ], + "properties": { + "x": { + "type": [ + "null", + "integer" + ] + }, + "width": { + "type": [ + "null", + "integer" + ] + }, + "y": { + "type": [ + "null", + "integer" + ] + }, + "height": { + "type": [ + "null", + "integer" + ] + } + } + } + } + }, + "defaultLocale": { + "type": [ + "null", + "object" + ], + "properties": { + "country": { + "type": [ + "null", + "string" + ] + }, + "language": { + "type": [ + "null", + "string" + ] + } + } + }, + "organizationType": { + "type": [ + "null", + "string" + ] + }, + "alternativeNames": { + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } + }, + "specialties": { + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } + }, + "staffCountRange": { + "type": [ + "null", + "string" + ] + }, + "localizedSpecialties": { + "type": [ + "null", + "array" + ], + "items": { + "items": {} + } + }, + "industries": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "name": { + "type": [ + "null", + "object" + ], + "properties": { + "localized": { + "type": [ + "null", + "object" + ], + "properties": { + "en_US": { + "type": [ + "null", + "string" + ] + } + } + }, + "preferredLocale": { + "type": [ + "null", + "object" + ], + "properties": { + "country": { + "type": [ + "null", + "string" + ] + }, + "language": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "primaryOrganizationType": { + "type": [ + "null", + "string" + ] + }, + "locations": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "locationType": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "object" + ], + "properties": { + "localized": { + "type": [ + "null", + "object" + ], + "properties": { + "en_US": { + "type": [ + "null", + "string" + ] + } + } + }, + "preferredLocale": { + "type": [ + "null", + "object" + ], + "properties": { + "country": { + "type": [ + "null", + "string" + ] + }, + "language": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "address": { + "type": [ + "null", + "object" + ], + "properties": { + "geographicArea": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "line1": { + "type": [ + "null", + "string" + ] + }, + "postalCode": { + "type": [ + "null", + "string" + ] + } + } + }, + "localizedDescription": { + "type": [ + "null", + "string" + ] + }, + "geoLocation": { + "type": [ + "null", + "string" + ] + }, + "streetAddressFieldState": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "localizedDescription": { + "type": [ + "null", + "string" + ] + }, + "$URN": { + "type": [ + "null", + "string" + ] + }, + "localizedWebsite": { + "type": [ + "null", + "string" + ] + }, + "logoV2": { + "type": [ + "null", + "object" + ], + "properties": { + "cropped": { + "type": [ + "null", + "string" + ] + }, + "original": { + "type": [ + "null", + "string" + ] + }, + "cropInfo": { + "type": [ + "null", + "object" + ], + "properties": { + "x": { + "type": [ + "null", + "integer" + ] + }, + "width": { + "type": [ + "null", + "integer" + ] + }, + "y": { + "type": [ + "null", + "integer" + ] + }, + "height": { + "type": [ + "null", + "integer" + ] + } + } + } + } + }, + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "required": [ + "row_id" + ] + } + }, + "x-infer-schema": true + }, + "key": [ + "/_meta/row_id" + ] + }, + { + "recommendedName": "follower_statistics", + "resourceConfig": { + "stream": "follower_statistics", + "syncMode": "full_refresh" + }, + "documentSchema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "followerCountsByStaffCountRange": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + } + }, + "followerCountsByFunction": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + } + }, + "followerCountsByAssociationType": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + } + }, + "followerCountsBySeniority": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + } + }, + "followerCountsByRegion": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + } + }, + "organizationalEntity": { + "type": [ + "null", + "string" + ] + }, + "followerCountsByCountry": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + } + }, + "followerCountsByIndustry": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + } + }, + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "required": [ + "row_id" + ] + } + }, + "x-infer-schema": true + }, + "key": [ + "/_meta/row_id" + ] + }, + { + "recommendedName": "share_statistics", + "resourceConfig": { + "stream": "share_statistics", + "syncMode": "full_refresh" + }, + "documentSchema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "totalShareStatistics": { + "type": [ + "null", + "object" + ], + "properties": { + "uniqueImpressionsCount": { + "type": [ + "null", + "integer" + ] + }, + "clickCount": { + "type": [ + "null", + "integer" + ] + }, + "engagement": { + "type": [ + "null", + "number" + ] + }, + "likeCount": { + "type": [ + "null", + "integer" + ] + }, + "commentCount": { + "type": [ + "null", + "integer" + ] + }, + "shareCount": { + "type": [ + "null", + "integer" + ] + }, + "commentMentionsCount": { + "type": [ + "null", + "integer" + ] + }, + "impressionCount": { + "type": [ + "null", + "integer" + ] + }, + "shareMentionsCount": { + "type": [ + "null", + "integer" + ] + } + } + }, + "organizationalEntity": { + "type": [ + "null", + "string" + ] + }, + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "required": [ + "row_id" + ] + } + }, + "x-infer-schema": true + }, + "key": [ + "/_meta/row_id" + ] + }, + { + "recommendedName": "total_follower_count", + "resourceConfig": { + "stream": "total_follower_count", + "syncMode": "full_refresh" + }, + "documentSchema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "firstDegreeSize": { + "type": [ + "null", + "integer" + ] + }, + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "required": [ + "row_id" + ] + } + }, + "x-infer-schema": true + }, + "key": [ + "/_meta/row_id" + ] + } +] \ No newline at end of file diff --git a/source-linkedin-pages/tests/snapshots/spec.stdout.json b/source-linkedin-pages/tests/snapshots/spec.stdout.json new file mode 100644 index 0000000000..3f002281f5 --- /dev/null +++ b/source-linkedin-pages/tests/snapshots/spec.stdout.json @@ -0,0 +1,124 @@ +{ + "protocol": 3032023, + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Linkedin Pages Spec", + "type": "object", + "required": [ + "org_id" + ], + "additionalProperties": true, + "properties": { + "org_id": { + "title": "Organization ID", + "type": "string", + "airbyte_secret": true, + "description": "Specify the Organization ID", + "examples": [ + "123456789" + ] + }, + "credentials": { + "title": "Authentication", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": [ + "client_id", + "client_secret", + "refresh_token" + ], + "properties": { + "auth_method": { + "type": "string", + "const": "oAuth2.0" + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The client ID of the LinkedIn developer application.", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client secret", + "description": "The client secret of the LinkedIn developer application.", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "title": "Refresh token", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "airbyte_secret": true + } + } + }, + { + "title": "Access token", + "type": "object", + "required": [ + "access_token" + ], + "properties": { + "auth_method": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + "title": "Access token", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "airbyte_secret": true + } + } + } + ] + } + } + }, + "resourceConfigSchema": { + "title": "ResourceConfig", + "description": "ResourceConfig encodes a configured resource stream", + "type": "object", + "properties": { + "stream": { + "title": "Stream", + "description": "Name of this stream", + "type": "string" + }, + "syncMode": { + "title": "Sync Mode", + "description": "Sync this resource incrementally, or fully refresh it every run", + "enum": [ + "full_refresh", + "incremental" + ], + "type": "string" + }, + "namespace": { + "title": "Namespace", + "description": "Enclosing schema namespace of this resource", + "type": "string" + }, + "cursorField": { + "title": "Cursor Field", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "stream", + "syncMode" + ], + "additionalProperties": false + }, + "documentationUrl": "https://docs.airbyte.com/integrations/sources/linkedin-pages/", + "resourcePathPointers": [ + "/namespace", + "/stream" + ] +} \ No newline at end of file diff --git a/source-linkedin-pages/tests/test_snapshots.py b/source-linkedin-pages/tests/test_snapshots.py new file mode 100644 index 0000000000..a12676b2b8 --- /dev/null +++ b/source-linkedin-pages/tests/test_snapshots.py @@ -0,0 +1,86 @@ +import json +import subprocess + + +def test_capture(request, snapshot): + snapshot.snapshot_dir = request.fspath.dirname + "/snapshots" + + result = subprocess.run( + [ + "flowctl", + "preview", + "--source", + request.fspath.dirname + "/../test.flow.yaml", + "--sessions", + "1", + "--delay", + "10s", + "--output", + "json", + ], + stdout=subprocess.PIPE, + text=True, + ) + assert result.returncode == 0 + + stdout = [] + for line in result.stdout.splitlines(): + stdout.append(json.loads(line)) + + snapshot.assert_match( + json.dumps(stdout, indent=2), + "capture.stdout.json", + ) + + +def test_discover(request, snapshot): + snapshot.snapshot_dir = request.fspath.dirname + "/snapshots" + + result = subprocess.run( + [ + "flowctl", + "raw", + "discover", + "--source", + request.fspath.dirname + "/../test.flow.yaml", + "--output", + "json", + "--emit-raw", + ], + stdout=subprocess.PIPE, + text=True, + ) + assert result.returncode == 0 + + stdout = [] + for line in result.stdout.splitlines(): + stdout.append(json.loads(line)) + + snapshot.assert_match( + json.dumps(stdout, indent=2), + "discover.stdout.json", + ) + + +def test_spec(request, snapshot): + snapshot.snapshot_dir = request.fspath.dirname + "/snapshots" + + result = subprocess.run( + [ + "flowctl", + "raw", + "spec", + "--output", + "json", + "--source", + request.fspath.dirname + "/../test.flow.yaml", + ], + stdout=subprocess.PIPE, + text=True, + ) + assert result.returncode == 0 + + snapshot.assert_match( + json.dumps(json.loads(result.stdout), indent=2), + "spec.stdout.json", + ) From a9a0aa6a504bc252170a0fedda2e7141981e27c4 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Thu, 28 Mar 2024 14:14:15 -0400 Subject: [PATCH 69/96] materialize-sql: forbid fields with no types We can't materialize fields with no types in SQL materializations, but we are currently recommending these fields by default which causes errors when the materialization tries to apply. This changes the validation constraints to produce FIELD_FORBIDDEN a constraint for fields with no types. --- materialize-sql/type_mapping.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/materialize-sql/type_mapping.go b/materialize-sql/type_mapping.go index 095ef8e201..1bcf3f6ae5 100644 --- a/materialize-sql/type_mapping.go +++ b/materialize-sql/type_mapping.go @@ -504,6 +504,9 @@ func (constrainter) NewConstraints(p *pf.Projection, deltaUpdates bool) *pm.Resp case p.IsRootDocumentProjection(): constraint.Type = pm.Response_Validated_Constraint_LOCATION_REQUIRED constraint.Reason = "The root document must be materialized" + case len(p.Inference.Types) == 0: + constraint.Type = pm.Response_Validated_Constraint_FIELD_FORBIDDEN + constraint.Reason = "Cannot materialize a field with no types" case p.Field == "_meta/op": constraint.Type = pm.Response_Validated_Constraint_LOCATION_RECOMMENDED constraint.Reason = "The operation type should usually be materialized" From bbb69b41b7cab0c883aa2d2db4c9f440a64c1107 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Thu, 28 Mar 2024 14:29:22 -0400 Subject: [PATCH 70/96] materialize-sql: don't log out the entire base64 encoded spec when doing apply Materialization specs can be quite enormous, and will often even exceed the maximum log size that flow allows. Rather than emitting this spec with the action description every time an apply runs, instead emit a placeholder value. The spec is stored in the database so we can always see what it is there for troubleshooting, and its hard to imagine a user getting anything other than confusion from the massive base64 string that we currently log out. --- materialize-sql/apply.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/materialize-sql/apply.go b/materialize-sql/apply.go index fb1f70428b..d674545f7c 100644 --- a/materialize-sql/apply.go +++ b/materialize-sql/apply.go @@ -146,6 +146,7 @@ func (a *sqlApplier) PutSpec(ctx context.Context, spec *pf.MaterializationSpec, panic(err) // Cannot fail to marshal. } + var description string var specUpdate MetaSpecsUpdate if a.endpoint.MetaSpecs != nil { // Insert or update the materialization specification. Both parameterized queries and @@ -168,6 +169,12 @@ func (a *sqlApplier) PutSpec(ctx context.Context, spec *pf.MaterializationSpec, a.endpoint.Literal(params[1].(string)), a.endpoint.Literal(params[2].(string)), } + var descriptionArgs = []interface{}{ + paramArgs[0], + a.endpoint.Literal(params[0].(string)), + a.endpoint.Literal("(a-base64-encoded-value)"), + a.endpoint.Literal(params[2].(string)), + } var q string if exists { @@ -179,9 +186,10 @@ func (a *sqlApplier) PutSpec(ctx context.Context, spec *pf.MaterializationSpec, specUpdate.ParameterizedQuery = fmt.Sprintf(q, paramArgs...) specUpdate.Parameters = params specUpdate.QueryString = fmt.Sprintf(q, queryStringArgs...) + description = fmt.Sprintf(q, descriptionArgs...) } - return specUpdate.QueryString, func(ctx context.Context) error { + return description, func(ctx context.Context) error { return a.client.PutSpec(ctx, specUpdate) }, nil } From 15db7347a04e93b8a2064858ad53fd7d4e30c4a5 Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 28 Mar 2024 16:21:42 -0300 Subject: [PATCH 71/96] Add source-linkedin-pages to test workflow --- .github/workflows/python.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 6b2261be90..9829aa529d 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -14,6 +14,7 @@ on: - "source-hubspot-native/**" - "source-hubspot/**" - "source-notion/**" + - "source-linkedin-pages/**" pull_request: branches: [main] paths: @@ -27,6 +28,7 @@ on: - "source-hubspot-native/**" - "source-hubspot/**" - "source-notion/**" + - "source-linkedin-pages/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -78,6 +80,10 @@ jobs: type: capture version: v2 usage_rate: "1.0" + - name: source-linkedin-pages + type: capture + version: v1 + usage_rate: "1.0" steps: - uses: actions/checkout@v4 From c18285f2f8b04c83d9943f0d0194c50fa3dca94e Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 28 Mar 2024 16:49:50 -0300 Subject: [PATCH 72/96] source-linkedin-pages: Fix dependencies --- source-linkedin-pages/poetry.lock | 196 +++++++++++++++++---------- source-linkedin-pages/pyproject.toml | 3 +- 2 files changed, 124 insertions(+), 75 deletions(-) diff --git a/source-linkedin-pages/poetry.lock b/source-linkedin-pages/poetry.lock index fb4742132f..483724b268 100644 --- a/source-linkedin-pages/poetry.lock +++ b/source-linkedin-pages/poetry.lock @@ -125,49 +125,49 @@ frozenlist = ">=1.1.0" [[package]] name = "airbyte-cdk" -version = "0.77.2" +version = "0.52.10" description = "A framework for writing Airbyte Connectors." optional = false -python-versions = "<4.0,>=3.9" +python-versions = ">=3.8" files = [ - {file = "airbyte_cdk-0.77.2-py3-none-any.whl", hash = "sha256:6dffbe0c4b3454a5cdd20525b4f1e9cfef2e80c005b6b30473fc5bf6f75af64e"}, - {file = "airbyte_cdk-0.77.2.tar.gz", hash = "sha256:84aeb27862a18e135c7bc3a5dfc363037665d428e7495e8824673f853adcca70"}, + {file = "airbyte-cdk-0.52.10.tar.gz", hash = "sha256:0daee950fe0d4453e6ceea2633090fc1d2144224e6f170b3c6cb4c6392811b47"}, + {file = "airbyte_cdk-0.52.10-py3-none-any.whl", hash = "sha256:366fd7bbbba317223edc1571d22b91c6f5bcff4ba65b3131e42f9b37e29932f4"}, ] [package.dependencies] -airbyte-protocol-models = "0.5.1" +airbyte-protocol-models = "0.4.2" backoff = "*" cachetools = "*" -Deprecated = ">=1.2,<1.3" +Deprecated = ">=1.2,<2.0" dpath = ">=2.0.1,<2.1.0" genson = "1.2.2" isodate = ">=0.6.1,<0.7.0" Jinja2 = ">=3.1.2,<3.2.0" -jsonref = ">=0.2,<0.3" +jsonref = ">=0.2,<1.0" jsonschema = ">=3.2.0,<3.3.0" -pendulum = "<3.0.0" +pendulum = "*" pydantic = ">=1.10.8,<2.0.0" -pyrate-limiter = ">=3.1.0,<3.2.0" python-dateutil = "*" -PyYAML = ">=6.0.1,<7.0.0" +PyYAML = ">=6.0.1" requests = "*" -requests_cache = "*" +requests-cache = "*" wcmatch = "8.4" [package.extras] -file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] -sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.19)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.19)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.19)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.19)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] name = "airbyte-protocol-models" -version = "0.5.1" +version = "0.4.2" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, - {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, ] [package.dependencies] @@ -646,13 +646,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jsonref" -version = "0.2" -description = "An implementation of JSON Reference for Python" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." optional = false -python-versions = "*" +python-versions = ">=3.3,<4.0" files = [ - {file = "jsonref-0.2-py3-none-any.whl", hash = "sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f"}, - {file = "jsonref-0.2.tar.gz", hash = "sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697"}, + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, ] [[package]] @@ -912,37 +912,102 @@ files = [ [[package]] name = "pendulum" -version = "2.1.2" +version = "3.0.0" description = "Python datetimes made easy" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.8" files = [ - {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, - {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, - {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, - {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, - {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, - {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, - {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, - {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, - {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, - {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, - {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, ] [package.dependencies] -python-dateutil = ">=2.6,<3.0" -pytzdata = ">=2020.1" +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] [[package]] name = "platformdirs" @@ -1103,21 +1168,6 @@ typing-extensions = ">=4.2.0" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] -[[package]] -name = "pyrate-limiter" -version = "3.1.1" -description = "Python Rate-Limiter using Leaky-Bucket Algorithm" -optional = false -python-versions = ">=3.8,<4.0" -files = [ - {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, - {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, -] - -[package.extras] -all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] -docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] - [[package]] name = "pyrsistent" version = "0.20.0" @@ -1207,17 +1257,6 @@ files = [ [package.dependencies] six = ">=1.5" -[[package]] -name = "pytzdata" -version = "2020.1" -description = "The Olson timezone database for Python." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, - {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, -] - [[package]] name = "pyyaml" version = "6.0.1" @@ -1393,6 +1432,17 @@ files = [ {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, ] +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + [[package]] name = "url-normalize" version = "1.4.3" @@ -1740,4 +1790,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "e5c243f78a01783701e27af42c46894d2e8df86d3a10a8582f2809bb65f81284" +content-hash = "50e49f7f1f9192f4b67a64b3e346857947b6fd95d10012b4c446164a52121b63" diff --git a/source-linkedin-pages/pyproject.toml b/source-linkedin-pages/pyproject.toml index 987a1eecd8..a58abfcb93 100644 --- a/source-linkedin-pages/pyproject.toml +++ b/source-linkedin-pages/pyproject.toml @@ -3,12 +3,11 @@ name = "source-linkedin-pages" version = "0.1.0" description = "" authors = ["Pedro "] -readme = "README.md" [tool.poetry.dependencies] python = "^3.11" estuary-cdk = { path = "../estuary-cdk", develop = true } -airbyte-cdk = "^0.77.2" +airbyte-cdk = "^0.52" [tool.poetry.group.dev.dependencies] pytest = "^8.1.1" From 01d731e2f8cbbcf2d51508be8f803f66053328b1 Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 28 Mar 2024 17:37:06 -0300 Subject: [PATCH 73/96] source-linkedin-pages: Add OAuth2 support --- .../source_linkedin_pages/__main__.py | 88 ++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/source-linkedin-pages/source_linkedin_pages/__main__.py b/source-linkedin-pages/source_linkedin_pages/__main__.py index a0684e25e7..5a42a26f45 100644 --- a/source-linkedin-pages/source_linkedin_pages/__main__.py +++ b/source-linkedin-pages/source_linkedin_pages/__main__.py @@ -1,13 +1,97 @@ import estuary_cdk.pydantic_polyfill # Must be first. # noqa import asyncio -from estuary_cdk import shim_airbyte_cdk +from estuary_cdk import shim_airbyte_cdk, flow from source_linkedin_pages import SourceLinkedinPages +import urllib.parse + + +def wrap_with_braces(body: str, count: int) -> str: + opening = "{" * count + closing = "}" * count + return f"{opening}{body}{closing}" + + +def urlencode_field(field: str) -> str: + return ( + f"{wrap_with_braces('#urlencode',2)}" + f"{wrap_with_braces(field,3)}" + f"{wrap_with_braces('/urlencode',2)}" + ) + + +def build_auth_url_template() -> str: + scopes = " ".join( + [ + "r_organization_followers", + "r_organization_social", + "rw_organization_admin", + "r_organization_social_feed", + "w_member_social", + "w_organization_social", + "r_basicprofile", + "w_organization_social_feed", + "w_member_social_feed", + "r_1st_connections_size", + ] + ) + query = urllib.parse.urlencode( + { + "client_id": wrap_with_braces("client_id", 3), + "redirect_uri": urlencode_field("redirect_uri"), + "response_type": "code", + "state": urlencode_field("state"), + "scope": scopes, + } + ) + url = urllib.parse.urlunsplit( + [ + "https", # scheme + "www.linkedin.com", # netloc + "/oauth/v2/authorization", # path + query, # query + "", # fragment + ] + ) + return url + + +def build_access_token_url_template() -> str: + query = urllib.parse.urlencode( + { + "grant_type": "authorization_code", + "code": urlencode_field("code"), + "client_id": wrap_with_braces("client_id", 3), + "client_secret": wrap_with_braces("client_secret", 3), + "redirect_uri": urlencode_field("redirect_uri"), + } + ) + url = urllib.parse.urlunsplit( + [ + "https", # scheme + "www.linkedin.com", # netloc + "/oauth/v2/accessToken", # path + query, # query + "", # fragment + ] + ) + return url + asyncio.run( shim_airbyte_cdk.CaptureShim( delegate=SourceLinkedinPages(), - oauth2=None, + oauth2=flow.OAuth2Spec( + provider="linkedin", + authUrlTemplate=build_auth_url_template(), + accessTokenUrlTemplate=build_access_token_url_template(), + accessTokenResponseMap={ + "access_token": "/access_token", + "refresh_token": "/refresh_token", + }, + accessTokenBody="", # Uses query arguments. + accessTokenHeaders={}, + ), schema_inference=True, ).serve() ) From d19d123a698568c94884539abc2f54282b86bf45 Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 28 Mar 2024 18:27:53 -0300 Subject: [PATCH 74/96] source-linkedin-pages: Add required field _meta to YAML and JSON schemas --- source-linkedin-pages/acmeCo/flow.yaml | 2 ++ .../acmeCo/follower_statistics.schema.yaml | 16 +++++++++------- .../acmeCo/organization_lookup.schema.yaml | 17 ++++++++++------- .../acmeCo/share_statistics.schema.yaml | 16 +++++++++------- .../schemas/follower_statistics.json | 11 +++++++++++ .../schemas/organization_lookup.json | 12 ++++++++++++ .../schemas/share_statistics.json | 11 +++++++++++ .../schemas/total_follower_count.json | 11 +++++++++++ 8 files changed, 75 insertions(+), 21 deletions(-) diff --git a/source-linkedin-pages/acmeCo/flow.yaml b/source-linkedin-pages/acmeCo/flow.yaml index 24fdc4def4..e47e2fbb2c 100644 --- a/source-linkedin-pages/acmeCo/flow.yaml +++ b/source-linkedin-pages/acmeCo/flow.yaml @@ -17,6 +17,8 @@ collections: $schema: "http://json-schema.org/draft-04/schema#" additionalProperties: true type: object + required: + - _meta properties: firstDegreeSize: type: diff --git a/source-linkedin-pages/acmeCo/follower_statistics.schema.yaml b/source-linkedin-pages/acmeCo/follower_statistics.schema.yaml index 696064c670..0711ff6ab0 100644 --- a/source-linkedin-pages/acmeCo/follower_statistics.schema.yaml +++ b/source-linkedin-pages/acmeCo/follower_statistics.schema.yaml @@ -2,7 +2,16 @@ $schema: "http://json-schema.org/draft-04/schema#" type: object additionalProperties: true +required: + - _meta properties: + _meta: + type: object + properties: + row_id: + type: integer + required: + - row_id followerCountsByStaffCountRange: type: - "null" @@ -70,11 +79,4 @@ properties: - "null" - object additionalProperties: true - _meta: - type: object - properties: - row_id: - type: integer - required: - - row_id x-infer-schema: true diff --git a/source-linkedin-pages/acmeCo/organization_lookup.schema.yaml b/source-linkedin-pages/acmeCo/organization_lookup.schema.yaml index 75766bd6fd..968ba40b35 100644 --- a/source-linkedin-pages/acmeCo/organization_lookup.schema.yaml +++ b/source-linkedin-pages/acmeCo/organization_lookup.schema.yaml @@ -2,7 +2,17 @@ $schema: "http://json-schema.org/draft-04/schema#" additionalProperties: true type: object +required: + - _meta properties: + _meta: + type: object + properties: + row_id: + type: integer + additionalProperties: true + required: + - row_id vanityName: type: - "null" @@ -323,11 +333,4 @@ properties: type: - "null" - integer - _meta: - type: object - properties: - row_id: - type: integer - required: - - row_id x-infer-schema: true diff --git a/source-linkedin-pages/acmeCo/share_statistics.schema.yaml b/source-linkedin-pages/acmeCo/share_statistics.schema.yaml index 1ad52b5769..237069cc17 100644 --- a/source-linkedin-pages/acmeCo/share_statistics.schema.yaml +++ b/source-linkedin-pages/acmeCo/share_statistics.schema.yaml @@ -2,7 +2,16 @@ $schema: "http://json-schema.org/draft-04/schema#" type: object additionalProperties: true +required: + - _meta properties: + _meta: + type: object + properties: + row_id: + type: integer + required: + - row_id totalShareStatistics: type: - "null" @@ -48,11 +57,4 @@ properties: type: - "null" - string - _meta: - type: object - properties: - row_id: - type: integer - required: - - row_id x-infer-schema: true diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json b/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json index 884277945d..a8cfe762b1 100644 --- a/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json +++ b/source-linkedin-pages/source_linkedin_pages/schemas/follower_statistics.json @@ -2,7 +2,18 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "additionalProperties": true, + "required": [ + "_meta" + ], "properties": { + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + } + }, "followerCountsByStaffCountRange": { "type": [ "null", diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json b/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json index cd3d37e33a..fa0a6468f0 100644 --- a/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json +++ b/source-linkedin-pages/source_linkedin_pages/schemas/organization_lookup.json @@ -2,7 +2,19 @@ "$schema": "http://json-schema.org/draft-04/schema#", "additionalProperties": true, "type": "object", + "required": [ + "_meta" + ], "properties": { + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "additionalProperties": true + }, "vanityName": { "type": [ "null", diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json b/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json index b88202f7ae..6b55f75424 100644 --- a/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json +++ b/source-linkedin-pages/source_linkedin_pages/schemas/share_statistics.json @@ -2,7 +2,18 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "additionalProperties": true, + "required": [ + "_meta" + ], "properties": { + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + } + }, "totalShareStatistics": { "type": [ "null", diff --git a/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json b/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json index 9458aa1088..2af3a44c2b 100644 --- a/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json +++ b/source-linkedin-pages/source_linkedin_pages/schemas/total_follower_count.json @@ -2,12 +2,23 @@ "$schema": "http://json-schema.org/draft-04/schema#", "additionalProperties": true, "type": "object", + "required": [ + "_meta" + ], "properties": { "firstDegreeSize": { "type": [ "null", "integer" ] + }, + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + } } } } \ No newline at end of file From 5f0982d41df52c4f5202c0c206589cdb03997b8b Mon Sep 17 00:00:00 2001 From: Pedro Date: Fri, 29 Mar 2024 15:49:59 -0300 Subject: [PATCH 75/96] source-linkedin-pages: Update the snapshot tests for the added OAuth2 support. --- .../tests/snapshots/discover.stdout.json | 79 +++++++++++-------- .../tests/snapshots/spec.stdout.json | 9 +++ 2 files changed, 55 insertions(+), 33 deletions(-) diff --git a/source-linkedin-pages/tests/snapshots/discover.stdout.json b/source-linkedin-pages/tests/snapshots/discover.stdout.json index 6a14998a9d..24304ed5b2 100644 --- a/source-linkedin-pages/tests/snapshots/discover.stdout.json +++ b/source-linkedin-pages/tests/snapshots/discover.stdout.json @@ -9,7 +9,22 @@ "$schema": "http://json-schema.org/draft-04/schema#", "additionalProperties": true, "type": "object", + "required": [ + "_meta" + ], "properties": { + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "additionalProperties": true, + "required": [ + "row_id" + ] + }, "vanityName": { "type": [ "null", @@ -499,17 +514,6 @@ } } } - }, - "_meta": { - "type": "object", - "properties": { - "row_id": { - "type": "integer" - } - }, - "required": [ - "row_id" - ] } }, "x-infer-schema": true @@ -528,7 +532,21 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "additionalProperties": true, + "required": [ + "_meta" + ], "properties": { + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "required": [ + "row_id" + ] + }, "followerCountsByStaffCountRange": { "type": [ "null", @@ -625,17 +643,6 @@ ], "additionalProperties": true } - }, - "_meta": { - "type": "object", - "properties": { - "row_id": { - "type": "integer" - } - }, - "required": [ - "row_id" - ] } }, "x-infer-schema": true @@ -654,7 +661,21 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "additionalProperties": true, + "required": [ + "_meta" + ], "properties": { + "_meta": { + "type": "object", + "properties": { + "row_id": { + "type": "integer" + } + }, + "required": [ + "row_id" + ] + }, "totalShareStatistics": { "type": [ "null", @@ -722,17 +743,6 @@ "null", "string" ] - }, - "_meta": { - "type": "object", - "properties": { - "row_id": { - "type": "integer" - } - }, - "required": [ - "row_id" - ] } }, "x-infer-schema": true @@ -751,6 +761,9 @@ "$schema": "http://json-schema.org/draft-04/schema#", "additionalProperties": true, "type": "object", + "required": [ + "_meta" + ], "properties": { "firstDegreeSize": { "type": [ diff --git a/source-linkedin-pages/tests/snapshots/spec.stdout.json b/source-linkedin-pages/tests/snapshots/spec.stdout.json index 3f002281f5..7859207e75 100644 --- a/source-linkedin-pages/tests/snapshots/spec.stdout.json +++ b/source-linkedin-pages/tests/snapshots/spec.stdout.json @@ -117,6 +117,15 @@ "additionalProperties": false }, "documentationUrl": "https://docs.airbyte.com/integrations/sources/linkedin-pages/", + "oauth2": { + "provider": "linkedin", + "authUrlTemplate": "https://www.linkedin.com/oauth/v2/authorization?client_id=%7B%7B%7Bclient_id%7D%7D%7D&redirect_uri=%7B%7B%23urlencode%7D%7D%7B%7B%7Bredirect_uri%7D%7D%7D%7B%7B%2Furlencode%7D%7D&response_type=code&state=%7B%7B%23urlencode%7D%7D%7B%7B%7Bstate%7D%7D%7D%7B%7B%2Furlencode%7D%7D&scope=r_organization_followers+r_organization_social+rw_organization_admin+r_organization_social_feed+w_member_social+w_organization_social+r_basicprofile+w_organization_social_feed+w_member_social_feed+r_1st_connections_size", + "accessTokenUrlTemplate": "https://www.linkedin.com/oauth/v2/accessToken?grant_type=authorization_code&code=%7B%7B%23urlencode%7D%7D%7B%7B%7Bcode%7D%7D%7D%7B%7B%2Furlencode%7D%7D&client_id=%7B%7B%7Bclient_id%7D%7D%7D&client_secret=%7B%7B%7Bclient_secret%7D%7D%7D&redirect_uri=%7B%7B%23urlencode%7D%7D%7B%7B%7Bredirect_uri%7D%7D%7D%7B%7B%2Furlencode%7D%7D", + "accessTokenResponseMap": { + "access_token": "/access_token", + "refresh_token": "/refresh_token" + } + }, "resourcePathPointers": [ "/namespace", "/stream" From de23c0fe5def5f60f1bada9786c312f0e83ff486 Mon Sep 17 00:00:00 2001 From: Pedro Date: Fri, 29 Mar 2024 15:53:37 -0300 Subject: [PATCH 76/96] source-linkedin-pages: Refactor test_capture function to remove snapshot and add explanatory comment. --- .../tests/snapshots/capture.stdout.json | 1192 ----------------- source-linkedin-pages/tests/test_snapshots.py | 18 +- 2 files changed, 6 insertions(+), 1204 deletions(-) delete mode 100644 source-linkedin-pages/tests/snapshots/capture.stdout.json diff --git a/source-linkedin-pages/tests/snapshots/capture.stdout.json b/source-linkedin-pages/tests/snapshots/capture.stdout.json deleted file mode 100644 index 9423b5509e..0000000000 --- a/source-linkedin-pages/tests/snapshots/capture.stdout.json +++ /dev/null @@ -1,1192 +0,0 @@ -[ - [ - "acmeCo/organization_lookup", - { - "$URN": "urn:li:organization:65266256", - "_meta": { - "op": "u", - "row_id": 0 - }, - "alternativeNames": [], - "coverPhotoV2": { - "cropInfo": { - "height": 212, - "width": 1256, - "x": 0, - "y": 97 - }, - "cropped": "urn:li:digitalmediaAsset:C561BAQGB95zZDFql-Q", - "original": "urn:li:digitalmediaAsset:C561BAQGB95zZDFql-Q" - }, - "defaultLocale": { - "country": "US", - "language": "en" - }, - "description": { - "localized": { - "en_US": "Estuary helps organizations gain real-time access to their data without having to manage infrastructure. Capture data from SaaS or technology sources, transform it and materialize it back into the same types of systems all with millisecond latency." - }, - "preferredLocale": { - "country": "US", - "language": "en" - } - }, - "foundedOn": { - "year": 2019 - }, - "groups": [], - "id": 65266256, - "industries": [ - "urn:li:industry:4" - ], - "localizedDescription": "Estuary helps organizations gain real-time access to their data without having to manage infrastructure. Capture data from SaaS or technology sources, transform it and materialize it back into the same types of systems all with millisecond latency.", - "localizedName": "Estuary", - "localizedSpecialties": [], - "localizedWebsite": "http://estuary.dev", - "locations": [ - { - "address": { - "city": "New York", - "country": "US", - "geographicArea": "NY", - "line1": " 244 Fifth Avenue", - "line2": "Suite 1277", - "postalCode": "10001" - }, - "description": { - "localized": { - "en_US": "Headquarters" - }, - "preferredLocale": { - "country": "US", - "language": "en" - } - }, - "geoLocation": "urn:li:geo:104207524", - "localizedDescription": "Headquarters", - "locationType": "HEADQUARTERS", - "streetAddressFieldState": "UNSET_OPT_OUT" - }, - { - "address": { - "city": "Columbus", - "country": "US", - "geographicArea": "Ohio", - "line1": "West State St", - "postalCode": "43215" - }, - "geoLocation": "urn:li:geo:100278071", - "locationType": "OTHER", - "streetAddressFieldState": "UNSET_OPT_OUT" - } - ], - "logoV2": { - "cropInfo": { - "height": 0, - "width": 0, - "x": 0, - "y": 0 - }, - "cropped": "urn:li:digitalmediaAsset:C560BAQGNlRBEM6qhzQ", - "original": "urn:li:digitalmediaAsset:C560BAQGNlRBEM6qhzQ" - }, - "name": { - "localized": { - "en_US": "Estuary" - }, - "preferredLocale": { - "country": "US", - "language": "en" - } - }, - "organizationType": "PRIVATELY_HELD", - "primaryOrganizationType": "NONE", - "specialties": [], - "staffCountRange": "SIZE_11_TO_50", - "vanityName": "estuary-tech", - "versionTag": "4106875789", - "website": { - "localized": { - "en_US": "http://estuary.dev" - }, - "preferredLocale": { - "country": "US", - "language": "en" - } - } - } - ], - [ - "acmeCo/follower_statistics", - { - "_meta": { - "op": "u", - "row_id": 0 - }, - "followerCountsByAssociationType": [ - { - "associationType": "EMPLOYEE", - "followerCounts": { - "organicFollowerCount": 24, - "paidFollowerCount": 0 - } - } - ], - "followerCountsByFunction": [ - { - "followerCounts": { - "organicFollowerCount": 1973, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:8" - }, - { - "followerCounts": { - "organicFollowerCount": 983, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:13" - }, - { - "followerCounts": { - "organicFollowerCount": 395, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:4" - }, - { - "followerCounts": { - "organicFollowerCount": 185, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:25" - }, - { - "followerCounts": { - "organicFollowerCount": 154, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:16" - }, - { - "followerCounts": { - "organicFollowerCount": 151, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:7" - }, - { - "followerCounts": { - "organicFollowerCount": 143, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:18" - }, - { - "followerCounts": { - "organicFollowerCount": 130, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:15" - }, - { - "followerCounts": { - "organicFollowerCount": 129, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:24" - }, - { - "followerCounts": { - "organicFollowerCount": 114, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:19" - }, - { - "followerCounts": { - "organicFollowerCount": 88, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:3" - }, - { - "followerCounts": { - "organicFollowerCount": 62, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:6" - }, - { - "followerCounts": { - "organicFollowerCount": 54, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:20" - }, - { - "followerCounts": { - "organicFollowerCount": 51, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:10" - }, - { - "followerCounts": { - "organicFollowerCount": 50, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:26" - }, - { - "followerCounts": { - "organicFollowerCount": 46, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:2" - }, - { - "followerCounts": { - "organicFollowerCount": 39, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:9" - }, - { - "followerCounts": { - "organicFollowerCount": 33, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:5" - }, - { - "followerCounts": { - "organicFollowerCount": 33, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:12" - }, - { - "followerCounts": { - "organicFollowerCount": 27, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:22" - }, - { - "followerCounts": { - "organicFollowerCount": 15, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:1" - }, - { - "followerCounts": { - "organicFollowerCount": 13, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:17" - }, - { - "followerCounts": { - "organicFollowerCount": 10, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:14" - }, - { - "followerCounts": { - "organicFollowerCount": 9, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:11" - }, - { - "followerCounts": { - "organicFollowerCount": 5, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:23" - }, - { - "followerCounts": { - "organicFollowerCount": 1, - "paidFollowerCount": 0 - }, - "function": "urn:li:function:21" - } - ], - "followerCountsByIndustry": [ - { - "followerCounts": { - "organicFollowerCount": 1318, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:4" - }, - { - "followerCounts": { - "organicFollowerCount": 1101, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:96" - }, - { - "followerCounts": { - "organicFollowerCount": 365, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:6" - }, - { - "followerCounts": { - "organicFollowerCount": 216, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:80" - }, - { - "followerCounts": { - "organicFollowerCount": 170, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:68" - }, - { - "followerCounts": { - "organicFollowerCount": 167, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:43" - }, - { - "followerCounts": { - "organicFollowerCount": 126, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:11" - }, - { - "followerCounts": { - "organicFollowerCount": 87, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3132" - }, - { - "followerCounts": { - "organicFollowerCount": 80, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:2458" - }, - { - "followerCounts": { - "organicFollowerCount": 78, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3102" - }, - { - "followerCounts": { - "organicFollowerCount": 69, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1673" - }, - { - "followerCounts": { - "organicFollowerCount": 68, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:102" - }, - { - "followerCounts": { - "organicFollowerCount": 66, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:5" - }, - { - "followerCounts": { - "organicFollowerCount": 56, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:27" - }, - { - "followerCounts": { - "organicFollowerCount": 56, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:41" - }, - { - "followerCounts": { - "organicFollowerCount": 52, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:14" - }, - { - "followerCounts": { - "organicFollowerCount": 49, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:8" - }, - { - "followerCounts": { - "organicFollowerCount": 47, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3128" - }, - { - "followerCounts": { - "organicFollowerCount": 45, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:70" - }, - { - "followerCounts": { - "organicFollowerCount": 44, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:118" - }, - { - "followerCounts": { - "organicFollowerCount": 43, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:132" - }, - { - "followerCounts": { - "organicFollowerCount": 41, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:75" - }, - { - "followerCounts": { - "organicFollowerCount": 41, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:46" - }, - { - "followerCounts": { - "organicFollowerCount": 41, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:129" - }, - { - "followerCounts": { - "organicFollowerCount": 40, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:84" - }, - { - "followerCounts": { - "organicFollowerCount": 39, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:42" - }, - { - "followerCounts": { - "organicFollowerCount": 36, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1285" - }, - { - "followerCounts": { - "organicFollowerCount": 34, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:104" - }, - { - "followerCounts": { - "organicFollowerCount": 33, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1725" - }, - { - "followerCounts": { - "organicFollowerCount": 32, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1862" - }, - { - "followerCounts": { - "organicFollowerCount": 32, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:106" - }, - { - "followerCounts": { - "organicFollowerCount": 31, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:105" - }, - { - "followerCounts": { - "organicFollowerCount": 31, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:116" - }, - { - "followerCounts": { - "organicFollowerCount": 31, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:69" - }, - { - "followerCounts": { - "organicFollowerCount": 30, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:100" - }, - { - "followerCounts": { - "organicFollowerCount": 29, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3106" - }, - { - "followerCounts": { - "organicFollowerCount": 29, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:103" - }, - { - "followerCounts": { - "organicFollowerCount": 29, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:47" - }, - { - "followerCounts": { - "organicFollowerCount": 28, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:12" - }, - { - "followerCounts": { - "organicFollowerCount": 28, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1999" - }, - { - "followerCounts": { - "organicFollowerCount": 27, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:57" - }, - { - "followerCounts": { - "organicFollowerCount": 24, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1594" - }, - { - "followerCounts": { - "organicFollowerCount": 24, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:44" - }, - { - "followerCounts": { - "organicFollowerCount": 24, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:53" - }, - { - "followerCounts": { - "organicFollowerCount": 23, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:90" - }, - { - "followerCounts": { - "organicFollowerCount": 22, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:15" - }, - { - "followerCounts": { - "organicFollowerCount": 22, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:30" - }, - { - "followerCounts": { - "organicFollowerCount": 22, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:112" - }, - { - "followerCounts": { - "organicFollowerCount": 22, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:99" - }, - { - "followerCounts": { - "organicFollowerCount": 21, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3130" - }, - { - "followerCounts": { - "organicFollowerCount": 20, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:17" - }, - { - "followerCounts": { - "organicFollowerCount": 19, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:28" - }, - { - "followerCounts": { - "organicFollowerCount": 18, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:52" - }, - { - "followerCounts": { - "organicFollowerCount": 18, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:19" - }, - { - "followerCounts": { - "organicFollowerCount": 18, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1157" - }, - { - "followerCounts": { - "organicFollowerCount": 17, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1855" - }, - { - "followerCounts": { - "organicFollowerCount": 17, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1644" - }, - { - "followerCounts": { - "organicFollowerCount": 17, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:67" - }, - { - "followerCounts": { - "organicFollowerCount": 16, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:124" - }, - { - "followerCounts": { - "organicFollowerCount": 16, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:130" - }, - { - "followerCounts": { - "organicFollowerCount": 16, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:10" - }, - { - "followerCounts": { - "organicFollowerCount": 15, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:59" - }, - { - "followerCounts": { - "organicFollowerCount": 15, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:7" - }, - { - "followerCounts": { - "organicFollowerCount": 14, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:34" - }, - { - "followerCounts": { - "organicFollowerCount": 14, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:91" - }, - { - "followerCounts": { - "organicFollowerCount": 13, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1" - }, - { - "followerCounts": { - "organicFollowerCount": 13, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:24" - }, - { - "followerCounts": { - "organicFollowerCount": 13, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:97" - }, - { - "followerCounts": { - "organicFollowerCount": 13, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3107" - }, - { - "followerCounts": { - "organicFollowerCount": 12, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:2081" - }, - { - "followerCounts": { - "organicFollowerCount": 12, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:25" - }, - { - "followerCounts": { - "organicFollowerCount": 12, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:9" - }, - { - "followerCounts": { - "organicFollowerCount": 12, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:48" - }, - { - "followerCounts": { - "organicFollowerCount": 12, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:113" - }, - { - "followerCounts": { - "organicFollowerCount": 11, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:92" - }, - { - "followerCounts": { - "organicFollowerCount": 11, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:31" - }, - { - "followerCounts": { - "organicFollowerCount": 11, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:140" - }, - { - "followerCounts": { - "organicFollowerCount": 11, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:137" - }, - { - "followerCounts": { - "organicFollowerCount": 11, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3242" - }, - { - "followerCounts": { - "organicFollowerCount": 10, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:32" - }, - { - "followerCounts": { - "organicFollowerCount": 10, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:63" - }, - { - "followerCounts": { - "organicFollowerCount": 10, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3099" - }, - { - "followerCounts": { - "organicFollowerCount": 10, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:73" - }, - { - "followerCounts": { - "organicFollowerCount": 10, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1339" - }, - { - "followerCounts": { - "organicFollowerCount": 9, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:94" - }, - { - "followerCounts": { - "organicFollowerCount": 9, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:126" - }, - { - "followerCounts": { - "organicFollowerCount": 9, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1713" - }, - { - "followerCounts": { - "organicFollowerCount": 9, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:1925" - }, - { - "followerCounts": { - "organicFollowerCount": 9, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:82" - }, - { - "followerCounts": { - "organicFollowerCount": 9, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:147" - }, - { - "followerCounts": { - "organicFollowerCount": 9, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3241" - }, - { - "followerCounts": { - "organicFollowerCount": 9, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:87" - }, - { - "followerCounts": { - "organicFollowerCount": 9, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:36" - }, - { - "followerCounts": { - "organicFollowerCount": 8, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3103" - }, - { - "followerCounts": { - "organicFollowerCount": 8, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:13" - }, - { - "followerCounts": { - "organicFollowerCount": 8, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:121" - }, - { - "followerCounts": { - "organicFollowerCount": 8, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:139" - }, - { - "followerCounts": { - "organicFollowerCount": 8, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3131" - }, - { - "followerCounts": { - "organicFollowerCount": 8, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:109" - }, - { - "followerCounts": { - "organicFollowerCount": 8, - "paidFollowerCount": 0 - }, - "industry": "urn:li:industry:3127" - } - ], - "followerCountsBySeniority": [ - { - "followerCounts": { - "organicFollowerCount": 2620, - "paidFollowerCount": 0 - }, - "seniority": "urn:li:seniority:3" - }, - { - "followerCounts": { - "organicFollowerCount": 1417, - "paidFollowerCount": 0 - }, - "seniority": "urn:li:seniority:4" - }, - { - "followerCounts": { - "organicFollowerCount": 277, - "paidFollowerCount": 0 - }, - "seniority": "urn:li:seniority:6" - }, - { - "followerCounts": { - "organicFollowerCount": 146, - "paidFollowerCount": 0 - }, - "seniority": "urn:li:seniority:10" - }, - { - "followerCounts": { - "organicFollowerCount": 136, - "paidFollowerCount": 0 - }, - "seniority": "urn:li:seniority:2" - }, - { - "followerCounts": { - "organicFollowerCount": 136, - "paidFollowerCount": 0 - }, - "seniority": "urn:li:seniority:5" - }, - { - "followerCounts": { - "organicFollowerCount": 133, - "paidFollowerCount": 0 - }, - "seniority": "urn:li:seniority:7" - }, - { - "followerCounts": { - "organicFollowerCount": 98, - "paidFollowerCount": 0 - }, - "seniority": "urn:li:seniority:8" - }, - { - "followerCounts": { - "organicFollowerCount": 45, - "paidFollowerCount": 0 - }, - "seniority": "urn:li:seniority:9" - }, - { - "followerCounts": { - "organicFollowerCount": 6, - "paidFollowerCount": 0 - }, - "seniority": "urn:li:seniority:1" - } - ], - "followerCountsByStaffCountRange": [ - { - "followerCounts": { - "organicFollowerCount": 905, - "paidFollowerCount": 0 - }, - "staffCountRange": "SIZE_10001_OR_MORE" - }, - { - "followerCounts": { - "organicFollowerCount": 773, - "paidFollowerCount": 0 - }, - "staffCountRange": "SIZE_11_TO_50" - }, - { - "followerCounts": { - "organicFollowerCount": 725, - "paidFollowerCount": 0 - }, - "staffCountRange": "SIZE_51_TO_200" - }, - { - "followerCounts": { - "organicFollowerCount": 552, - "paidFollowerCount": 0 - }, - "staffCountRange": "SIZE_1001_TO_5000" - }, - { - "followerCounts": { - "organicFollowerCount": 513, - "paidFollowerCount": 0 - }, - "staffCountRange": "SIZE_2_TO_10" - }, - { - "followerCounts": { - "organicFollowerCount": 394, - "paidFollowerCount": 0 - }, - "staffCountRange": "SIZE_201_TO_500" - }, - { - "followerCounts": { - "organicFollowerCount": 306, - "paidFollowerCount": 0 - }, - "staffCountRange": "SIZE_501_TO_1000" - }, - { - "followerCounts": { - "organicFollowerCount": 223, - "paidFollowerCount": 0 - }, - "staffCountRange": "SIZE_5001_TO_10000" - }, - { - "followerCounts": { - "organicFollowerCount": 88, - "paidFollowerCount": 0 - }, - "staffCountRange": "SIZE_1" - } - ], - "organizationalEntity": "urn:li:organization:65266256" - } - ], - [ - "acmeCo/share_statistics", - { - "_meta": { - "op": "u", - "row_id": 0 - }, - "organizationalEntity": "urn:li:organization:65266256", - "totalShareStatistics": { - "clickCount": 1584, - "commentCount": 39, - "commentMentionsCount": 48, - "engagement": 0.042974090932691805, - "impressionCount": 57779, - "likeCount": 819, - "shareCount": 41, - "shareMentionsCount": 78, - "uniqueImpressionsCount": 21282 - } - } - ], - [ - "acmeCo/total_follower_count", - { - "_meta": { - "op": "u", - "row_id": 0 - }, - "firstDegreeSize": 6284 - } - ] -] \ No newline at end of file diff --git a/source-linkedin-pages/tests/test_snapshots.py b/source-linkedin-pages/tests/test_snapshots.py index a12676b2b8..82e0f1e9b6 100644 --- a/source-linkedin-pages/tests/test_snapshots.py +++ b/source-linkedin-pages/tests/test_snapshots.py @@ -2,9 +2,12 @@ import subprocess -def test_capture(request, snapshot): - snapshot.snapshot_dir = request.fspath.dirname + "/snapshots" - +def test_capture(request): + """ + This test doesn't have a snapshot + because the output data changes too frequently. + Almost all field values changes between test executions. + """ result = subprocess.run( [ "flowctl", @@ -23,15 +26,6 @@ def test_capture(request, snapshot): ) assert result.returncode == 0 - stdout = [] - for line in result.stdout.splitlines(): - stdout.append(json.loads(line)) - - snapshot.assert_match( - json.dumps(stdout, indent=2), - "capture.stdout.json", - ) - def test_discover(request, snapshot): snapshot.snapshot_dir = request.fspath.dirname + "/snapshots" From f54d9a8297d94e136c4cf90195e01d92b01b372f Mon Sep 17 00:00:00 2001 From: Will Baker Date: Mon, 1 Apr 2024 16:59:27 -0400 Subject: [PATCH 77/96] snowflake: update go driver from v1.7.2 to v1.9.0 --- go.mod | 6 +++--- go.sum | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index cd57453ca6..b451021c25 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/go-mysql-org/go-mysql v1.5.0 github.com/go-sql-driver/mysql v1.6.0 github.com/gogo/protobuf v1.3.2 + github.com/golang-jwt/jwt/v5 v5.2.0 github.com/google/uuid v1.3.1 github.com/iancoleman/orderedmap v0.2.0 github.com/invopop/jsonschema v0.5.0 @@ -50,7 +51,7 @@ require ( github.com/segmentio/encoding v0.3.6 github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed github.com/sirupsen/logrus v1.9.0 - github.com/snowflakedb/gosnowflake v1.7.2 + github.com/snowflakedb/gosnowflake v1.9.0 github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.16.0 github.com/trinodb/trino-go-client v0.313.0 @@ -91,7 +92,7 @@ require ( github.com/andybalholm/brotli v1.0.5 // indirect github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect github.com/apache/arrow/go/v12 v12.0.1 // indirect - github.com/apache/arrow/go/v14 v14.0.2 // indirect + github.com/apache/arrow/go/v15 v15.0.0 // indirect github.com/apache/thrift v0.17.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect @@ -126,7 +127,6 @@ require ( github.com/go-jose/go-jose/v3 v3.0.1 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/glog v1.1.2 // indirect diff --git a/go.sum b/go.sum index 1f129ba936..2251738b08 100644 --- a/go.sum +++ b/go.sum @@ -150,8 +150,8 @@ github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6IC github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs= github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg= github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= -github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= -github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= +github.com/apache/arrow/go/v15 v15.0.0 h1:1zZACWf85oEZY5/kd9dsQS7i+2G5zVQcbKTHgslqHNA= +github.com/apache/arrow/go/v15 v15.0.0/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= @@ -849,8 +849,8 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/snowflakedb/gosnowflake v1.7.2 h1:HRSwva8YXC64WUppfmHcMNVVzSE1+EwXXaJxgS0EkTo= -github.com/snowflakedb/gosnowflake v1.7.2/go.mod h1:03tW856vc3ceM4rJuj7KO4dzqN7qoezTm+xw7aPIIFo= +github.com/snowflakedb/gosnowflake v1.9.0 h1:s2ZdwFxFfpqwa5CqlhnzRESnLmwU3fED6zyNOJHFBQA= +github.com/snowflakedb/gosnowflake v1.9.0/go.mod h1:4ZgHxVf2OKwecx07WjfyAMr0gn8Qj4yvwAo68Og8wsU= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= From 9e5be6765fd573ed5a98b23001d4fa1dad765e90 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Wed, 27 Mar 2024 15:44:28 -0400 Subject: [PATCH 78/96] airbyte shim: initialize bindingStateV1 if field is not set When a connector is first created, it will be opened with an empty state. Our pydantic serialization will only serialize fields that are explicitly set. Parsing the empty state object was resulting in the `bindingStateV1` field _not_ being explicitly set, even if later actions in the airbyte shim modified it. This explicitly sets the `bindingStateV1` field if it isn't already so that the updated state can be output in a checkpoint. --- estuary-cdk/estuary_cdk/shim_airbyte_cdk.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py b/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py index c4016f1442..e23c5e2700 100644 --- a/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py +++ b/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py @@ -268,6 +268,12 @@ async def _run( ] airbyte_catalog = ConfiguredAirbyteCatalog(streams=airbyte_streams) + if "bindingStateV1" not in connector_state.__fields_set__: + # Initialize the top-level state object so that it is properly serialized if this is an + # "empty" state, which occurs for a brand new task that has never emitted any + # checkpoints. + connector_state.__setattr__("bindingStateV1", {}) + # Index of Airbyte (namespace, stream) => ResourceState. # Use `setdefault()` to initialize ResourceState if it's not already part of `connector_state`. index: dict[tuple[str | None, str], tuple[int, ResourceState]] = { From 86662aec7f6626386a13bee4c23bdba522eaaa70 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Wed, 27 Mar 2024 16:26:11 -0400 Subject: [PATCH 79/96] airbyte shim: ignore extra fields in persisted state For migrations of pre-existing connectors, they will have persisted a state not in the canonical form keyed by the top-level `bindingStateV1` key. This was causing irrecoverable errors due to the state model not allowing any extra fields. Instead we'll just ignore extra fields, which will result in migrated tasks starting over from the beginning, but at least they'll work. --- estuary-cdk/estuary_cdk/shim_airbyte_cdk.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py b/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py index e23c5e2700..f04bf3ca3b 100644 --- a/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py +++ b/estuary-cdk/estuary_cdk/shim_airbyte_cdk.py @@ -78,8 +78,15 @@ class ResourceState(common.BaseResourceState, extra="forbid"): """state is a dict encoding of AirbyteStateMessage""" -ConnectorState = common.ConnectorState[ResourceState] -"""Use the common.ConnectorState shape with ResourceState""" +class ConnectorState(common.ConnectorState[ResourceState], extra="ignore"): + """ConnectorState represents a number of ResourceStates, keyed by binding state key. + + Top-level fields other than bindingStateV1 are ignored, to allow for a lossy migration from + states that existed prior to adopting this convection. Connectors transitioning in this way will + effectively start over from the beginning. + """ + + bindingStateV1: dict[str, ResourceState] = {} class Document(common.BaseDocument, extra="allow"): From 073125b0e4f6fdeed8b3880f96eb2948688bc5e1 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Mon, 1 Apr 2024 18:25:54 -0500 Subject: [PATCH 80/96] source-firestore: Error message improvements We fairly often see a fatal error message from `source-firestore` which reads simply: unexpected TargetChange.REMOVE: The datastore operation timed out, or the data was temporarily unavailable And we should really be mentioning the name of the collection on which this error occurred. This commit should fix that, and also makes sure that we mention the collection name on backfill errors as well. --- source-firestore/pull.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source-firestore/pull.go b/source-firestore/pull.go index 8d8bbd623f..16493dc8e6 100644 --- a/source-firestore/pull.go +++ b/source-firestore/pull.go @@ -433,14 +433,20 @@ func (c *capture) Run(ctx context.Context) error { var collectionID, startTime = collectionID, startTime // Copy the loop variables for each closure log.WithField("collection", collectionID).Debug("starting worker") eg.Go(func() error { - return c.StreamChanges(ctx, rpcClient, collectionID, startTime) + if err := c.StreamChanges(ctx, rpcClient, collectionID, startTime); err != nil { + return fmt.Errorf("error streaming changes for collection %q: %w", collectionID, err) + } + return nil }) } for collectionID, resumeState := range backfillCollections { var collectionID, resumeState = collectionID, resumeState // Copy loop variables for each closure log.WithField("collection", collectionID).Debug("starting backfill worker") eg.Go(func() error { - return c.BackfillAsync(ctx, libraryClient, collectionID, resumeState) + if err := c.BackfillAsync(ctx, libraryClient, collectionID, resumeState); err != nil { + return fmt.Errorf("error backfilling collection %q: %w", collectionID, err) + } + return nil }) } defer log.Info("capture terminating") From feed6c09d3ad772abbf74a1c86b62d3649692de9 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Tue, 2 Apr 2024 10:25:46 -0500 Subject: [PATCH 81/96] source-firestore: Log more information about backfill states --- source-firestore/pull.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source-firestore/pull.go b/source-firestore/pull.go index 16493dc8e6..1da13b45a1 100644 --- a/source-firestore/pull.go +++ b/source-firestore/pull.go @@ -368,7 +368,15 @@ func (c *capture) Run(ctx context.Context) error { } if resourceState.Backfill == nil || resourceState.Backfill.Completed { // Do nothing when no backfill is required - } else if resumeState, ok := backfillCollections[collectionID]; !ok { + continue + } + log.WithFields(log.Fields{ + "resource": resourceState.path, + "collection": collectionID, + "startAfter": resourceState.Backfill.StartAfter, + "cursor": resourceState.Backfill.Cursor, + }).Debug("backfill required for binding") + if resumeState, ok := backfillCollections[collectionID]; !ok { backfillCollections[collectionID] = resourceState.Backfill } else if !resumeState.Equal(resourceState.Backfill) { return fmt.Errorf("internal error: backfill state mismatch for resource %q with collection ID %q", resourceState.path, collectionID) From 2abe56cbd2c32551d44b001d12cbb47ebbe1e5fa Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Tue, 2 Apr 2024 10:26:03 -0500 Subject: [PATCH 82/96] source-firestore: Retry unexpected TargetChange.REMOVE events --- source-firestore/pull.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source-firestore/pull.go b/source-firestore/pull.go index 1da13b45a1..6665d2d5ce 100644 --- a/source-firestore/pull.go +++ b/source-firestore/pull.go @@ -758,6 +758,7 @@ func (c *capture) StreamChanges(ctx context.Context, client *firestore_v1.Client return fmt.Errorf("unexpected target ID %d", tc.TargetIds[0]) } case firestore_pb.TargetChange_REMOVE: + listenClient = nil if catchupStreaming && time.Since(catchupStarted) > 5*time.Minute { logEntry.WithField("docs", numDocuments).Warn("replication failed to catch up in time, skipping to latest changes (go.estuary.dev/YRDsKd)") if checkpointJSON, err := c.State.MarkInconsistent(collectionID); err != nil { @@ -769,11 +770,10 @@ func (c *capture) StreamChanges(ctx context.Context, client *firestore_v1.Client logEntry.Fatal("forcing connector restart to establish consistency") }) target.ResumeType = &firestore_pb.Target_ReadTime{ReadTime: timestamppb.New(time.Now())} - listenClient = nil } else if tc.Cause != nil { - return fmt.Errorf("unexpected TargetChange.REMOVE: %v", tc.Cause.Message) + logEntry.WithField("cause", tc.Cause.Message).Warn("unexpected TargetChange.REMOVE") } else { - return fmt.Errorf("unexpected TargetChange.REMOVE") + logEntry.Warn("unexpected TargetChange.REMOVE") } case firestore_pb.TargetChange_CURRENT: if log.IsLevelEnabled(log.TraceLevel) { From 6db7e4a98da653a2ea91d4dac60d544028c66456 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Tue, 2 Apr 2024 11:17:13 -0500 Subject: [PATCH 83/96] source-firestore: Restart backfills on state mismatch Firestore's data model is kind of insane, internally. In order to cope with the arbitrarily-nested nature of things they have a concept of a "Collection Group" which is the set of all documents with the same final (non-ID) path element, so for instance `users/123/messages` and `groups/456/messages` are both found in the `messages` collection group. We use this in our backfills (and then process the document path ourselves to figure out which binding, if any, a document should be captured under). This means that multiple bindings for distinct resource paths may actually be combined in a single backfill, so we have logic to recombine the per-resource backfill states into a single backfill state and update all impacted resource states as one. In general this works, but there are some rough edges if you add a new binding (or equivalently, increment the backfill counter) with a collection group that's the same as other ongoing backfills which have made partial progress. Previously this resulted in an error, because of the mismatch, with the implied expectation that the user needs to know to backfill all bindings with the same final path element. Which is not very user- friendly at all. The ideal behavior given how collection groups work is to restart the backfills of all impacted bindings when this occurs, so after this commit that's what we'll do. --- source-firestore/pull.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source-firestore/pull.go b/source-firestore/pull.go index 6665d2d5ce..c1ed3bf445 100644 --- a/source-firestore/pull.go +++ b/source-firestore/pull.go @@ -379,7 +379,18 @@ func (c *capture) Run(ctx context.Context) error { if resumeState, ok := backfillCollections[collectionID]; !ok { backfillCollections[collectionID] = resourceState.Backfill } else if !resumeState.Equal(resourceState.Backfill) { - return fmt.Errorf("internal error: backfill state mismatch for resource %q with collection ID %q", resourceState.path, collectionID) + log.WithFields(log.Fields{ + "resource": resourceState.path, + "collection": collectionID, + }).Warn("backfill state mismatch, restarting all impacted collections") + + resumeState.Cursor = "" + resumeState.MTime = time.Time{} + if resumeState.StartAfter.After(resourceState.Backfill.StartAfter) { + // Take the minimum StartAfter time across all collections so that we begin + // backfilling the new one as soon as possible. + resumeState.StartAfter = resourceState.Backfill.StartAfter + } } } From d6a5992b5fd116d6856aabe3fc000c80d89023c1 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 3 Apr 2024 12:27:47 +0100 Subject: [PATCH 84/96] m-snowflake: increase retry count and delay --- materialize-snowflake/snowflake.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/materialize-snowflake/snowflake.go b/materialize-snowflake/snowflake.go index e2b931e047..06e33900f7 100644 --- a/materialize-snowflake/snowflake.go +++ b/materialize-snowflake/snowflake.go @@ -619,8 +619,9 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error // until they have all been successful, or an error has been thrown // If we see no results from the REST API for `maxTries` iterations, then we - // fallback to asking the `COPY_HISTORY` table. - var maxTries = 10 + // fallback to asking the `COPY_HISTORY` table. We allow up to 5 minutes for results to show up in the REST API. + var maxTries = 60 + var retryDelaySeconds = 5 * time.Second for pipeName, pipe := range pipes { for tries := 0; tries < maxTries; tries++ { // We first try to check the status of pipes using the REST API's insertReport @@ -652,7 +653,7 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error // We try `maxTries` times since it may take some time for the REST API // to reflect the new pipe requests if tries < maxTries-1 { - time.Sleep(5 * time.Second) + time.Sleep(retryDelaySeconds) continue } @@ -688,7 +689,7 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error // If items are still in progress, we continue trying to fetch their results if hasItemsInProgress { tries-- - time.Sleep(2 * time.Second) + time.Sleep(retryDelaySeconds) continue } @@ -719,7 +720,7 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error d.deleteFiles(ctx, []string{pipe.dir}) break } else { - time.Sleep(5 * time.Second) + time.Sleep(retryDelaySeconds) } } } From af53bb5fe19b0a7b50af2c8617be53f0260c0abc Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Wed, 3 Apr 2024 12:46:42 +0100 Subject: [PATCH 85/96] m-snowflake: SNOWFLAKE_PRIVATE_KEY is not mandatory --- tests/materialize/materialize-snowflake/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/materialize/materialize-snowflake/setup.sh b/tests/materialize/materialize-snowflake/setup.sh index b4deb502af..78823f0a8d 100755 --- a/tests/materialize/materialize-snowflake/setup.sh +++ b/tests/materialize/materialize-snowflake/setup.sh @@ -14,7 +14,7 @@ export SNOWFLAKE_AUTH_TYPE="${SNOWFLAKE_AUTH_TYPE}" # if auth type is user_password export SNOWFLAKE_USER="${SNOWFLAKE_USER:-}" export SNOWFLAKE_PASSWORD="${SNOWFLAKE_PASSWORD:-}" -if [ -z "${${SNOWFLAKE_PRIVATE_KEY+x}}" ]; then +if [ -n "${SNOWFLAKE_PRIVATE_KEY:-}" ]; then # if auth type is jwt export SNOWFLAKE_PRIVATE_KEY="$(cat ${SNOWFLAKE_PRIVATE_KEY:-} | jq -sR . | sed -e 's/^"//' -e 's/"$//')" fi From ba3ed02f6b716246110bfb4435111cdf6136dacd Mon Sep 17 00:00:00 2001 From: Pedro Date: Wed, 3 Apr 2024 13:32:23 -0300 Subject: [PATCH 86/96] source-linkedin-pages: Update spec.json with new documentation URL --- .../source_linkedin_pages/spec.json | 26 +++++++++++++------ .../tests/snapshots/spec.stdout.json | 6 ++--- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/source-linkedin-pages/source_linkedin_pages/spec.json b/source-linkedin-pages/source_linkedin_pages/spec.json index dfb3a0460e..1ed45f0913 100644 --- a/source-linkedin-pages/source_linkedin_pages/spec.json +++ b/source-linkedin-pages/source_linkedin_pages/spec.json @@ -1,10 +1,12 @@ { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/linkedin-pages/", + "documentationUrl": "https://docs.estuary.dev/reference/Connectors/capture-connectors/linkedin-pages/", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Linkedin Pages Spec", "type": "object", - "required": ["org_id"], + "required": [ + "org_id" + ], "additionalProperties": true, "properties": { "org_id": { @@ -12,7 +14,9 @@ "type": "string", "airbyte_secret": true, "description": "Specify the Organization ID", - "examples": ["123456789"] + "examples": [ + "123456789" + ] }, "credentials": { "title": "Authentication", @@ -21,7 +25,11 @@ { "type": "object", "title": "OAuth2.0", - "required": ["client_id", "client_secret", "refresh_token"], + "required": [ + "client_id", + "client_secret", + "refresh_token" + ], "properties": { "auth_method": { "type": "string", @@ -42,7 +50,7 @@ "refresh_token": { "type": "string", "title": "Refresh token", - "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://docs.estuary.dev/reference/Connectors/capture-connectors/linkedin-pages/.", "airbyte_secret": true } } @@ -50,7 +58,9 @@ { "title": "Access token", "type": "object", - "required": ["access_token"], + "required": [ + "access_token" + ], "properties": { "auth_method": { "type": "string", @@ -59,7 +69,7 @@ "access_token": { "type": "string", "title": "Access token", - "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://docs.estuary.dev/reference/Connectors/capture-connectors/linkedin-pages/", "airbyte_secret": true } } @@ -68,4 +78,4 @@ } } } -} +} \ No newline at end of file diff --git a/source-linkedin-pages/tests/snapshots/spec.stdout.json b/source-linkedin-pages/tests/snapshots/spec.stdout.json index 7859207e75..2ab4359d50 100644 --- a/source-linkedin-pages/tests/snapshots/spec.stdout.json +++ b/source-linkedin-pages/tests/snapshots/spec.stdout.json @@ -50,7 +50,7 @@ "refresh_token": { "type": "string", "title": "Refresh token", - "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://.com/integrations/sources/linkedin-pages/", "airbyte_secret": true } } @@ -69,7 +69,7 @@ "access_token": { "type": "string", "title": "Access token", - "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours.", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://docs.estuary.dev/reference/Connectors/capture-connectors/linkedin-pages/", "airbyte_secret": true } } @@ -116,7 +116,7 @@ ], "additionalProperties": false }, - "documentationUrl": "https://docs.airbyte.com/integrations/sources/linkedin-pages/", + "documentationUrl": "https://docs.estuary.dev/reference/Connectors/capture-connectors/linkedin-pages/", "oauth2": { "provider": "linkedin", "authUrlTemplate": "https://www.linkedin.com/oauth/v2/authorization?client_id=%7B%7B%7Bclient_id%7D%7D%7D&redirect_uri=%7B%7B%23urlencode%7D%7D%7B%7B%7Bredirect_uri%7D%7D%7D%7B%7B%2Furlencode%7D%7D&response_type=code&state=%7B%7B%23urlencode%7D%7D%7B%7B%7Bstate%7D%7D%7D%7B%7B%2Furlencode%7D%7D&scope=r_organization_followers+r_organization_social+rw_organization_admin+r_organization_social_feed+w_member_social+w_organization_social+r_basicprofile+w_organization_social_feed+w_member_social_feed+r_1st_connections_size", From 75fe4a201a5089901a397d1e0d634d0c8b4ede70 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Wed, 3 Apr 2024 11:30:38 -0500 Subject: [PATCH 87/96] source-firestore: Only retry TargetChange.REMOVE after 60s This *probably* isn't necessary, but one of the challenges with Firestore change streaming is that it's actually not super hard to end up in a permanent failure loop where you keep reading the same documents over and over and accidentally rack up large bills. In other places where we retry after a failure status we generally wait 60 seconds before trying again, and it's probably prudent to do that for the newly added `TargetChange.REMOVE` retry logic as well. --- source-firestore/pull.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source-firestore/pull.go b/source-firestore/pull.go index c1ed3bf445..6054526b00 100644 --- a/source-firestore/pull.go +++ b/source-firestore/pull.go @@ -783,8 +783,10 @@ func (c *capture) StreamChanges(ctx context.Context, client *firestore_v1.Client target.ResumeType = &firestore_pb.Target_ReadTime{ReadTime: timestamppb.New(time.Now())} } else if tc.Cause != nil { logEntry.WithField("cause", tc.Cause.Message).Warn("unexpected TargetChange.REMOVE") + time.Sleep(retryInterval) } else { logEntry.Warn("unexpected TargetChange.REMOVE") + time.Sleep(retryInterval) } case firestore_pb.TargetChange_CURRENT: if log.IsLevelEnabled(log.TraceLevel) { From 4bba9633a82639defc4b9c707dab0f89b92b45d5 Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Wed, 3 Apr 2024 13:07:54 -0500 Subject: [PATCH 88/96] source-firestore: Handle catchup failures after any time interval When streaming replication events from Firestore, sometimes the streaming for a particular high-volume collection gets stuck in a state where we open the stream, and then before becoming caught up Firestore sends us a `TargetChange.REMOVE` event. In such cases it is unlikely that we will ever succeed on a subsequent retry, so we have logic to give up and restart streaming from "now" and go back and re-backfill the collection later. In the past we've always observed this happening after exactly 10 minutes, so the condition for triggering this recovery behavior includes spending >5m streaming change events before getting cut off. However now we're seeing basically the exact same pattern of behavior occurring after less than two minutes, and the desired response is the same. So the appropriate fix here is to just remove the time threshold entirely. After this commit, any time we receive a `TargetChange.REMOVE` event before the stream catches up, we'll immediately give up and jump ahead to the latest changes for the collection. --- source-firestore/pull.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source-firestore/pull.go b/source-firestore/pull.go index 6054526b00..4ffb45aa95 100644 --- a/source-firestore/pull.go +++ b/source-firestore/pull.go @@ -678,7 +678,6 @@ func (c *capture) StreamChanges(ctx context.Context, client *firestore_v1.Client var listenClient firestore_pb.Firestore_ListenClient var numRestarts, numDocuments int var isCurrent, catchupStreaming bool - var catchupStarted time.Time for { if listenClient == nil { var err error @@ -708,7 +707,6 @@ func (c *capture) StreamChanges(ctx context.Context, client *firestore_v1.Client } }() } - catchupStarted = time.Now() } resp, err := listenClient.Recv() @@ -770,8 +768,8 @@ func (c *capture) StreamChanges(ctx context.Context, client *firestore_v1.Client } case firestore_pb.TargetChange_REMOVE: listenClient = nil - if catchupStreaming && time.Since(catchupStarted) > 5*time.Minute { - logEntry.WithField("docs", numDocuments).Warn("replication failed to catch up in time, skipping to latest changes (go.estuary.dev/YRDsKd)") + if catchupStreaming { + logEntry.WithField("docs", numDocuments).Warn("replication failed to catch up, skipping to latest changes (go.estuary.dev/YRDsKd)") if checkpointJSON, err := c.State.MarkInconsistent(collectionID); err != nil { return err } else if err := c.Output.Checkpoint(checkpointJSON, true); err != nil { From cd7c1b12ed2aa450e24953e2d5df18143f12e4ad Mon Sep 17 00:00:00 2001 From: Luishfs Date: Thu, 4 Apr 2024 14:15:58 -0300 Subject: [PATCH 89/96] Adding usage rate to build-local script --- build-local.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build-local.sh b/build-local.sh index 40545da710..4b33d41c01 100755 --- a/build-local.sh +++ b/build-local.sh @@ -26,6 +26,7 @@ docker buildx build \ --platform linux/amd64 \ --build-arg CONNECTOR_NAME="$1" \ --build-arg CONNECTOR_TYPE="$CONNECTOR_TYPE" \ + --build-arg="USAGE_RATE=1.0" \ --load \ -t ghcr.io/estuary/"$1":local \ -f "$DOCKERFILE" \ From 2abb9fc6c39b747dd7191a3cbd592bdf21f0aac7 Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 4 Apr 2024 18:40:20 -0300 Subject: [PATCH 90/96] source-linkedin-pages: Fix LinkedIn documentation and OAuth URLs --- .../source_linkedin_pages/__main__.py | 36 +++++++++---------- .../source_linkedin_pages/spec.json | 6 ++-- .../tests/snapshots/spec.stdout.json | 10 +++--- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/source-linkedin-pages/source_linkedin_pages/__main__.py b/source-linkedin-pages/source_linkedin_pages/__main__.py index 5a42a26f45..6d0bc61f86 100644 --- a/source-linkedin-pages/source_linkedin_pages/__main__.py +++ b/source-linkedin-pages/source_linkedin_pages/__main__.py @@ -35,15 +35,16 @@ def build_auth_url_template() -> str: "r_1st_connections_size", ] ) - query = urllib.parse.urlencode( - { - "client_id": wrap_with_braces("client_id", 3), - "redirect_uri": urlencode_field("redirect_uri"), - "response_type": "code", - "state": urlencode_field("state"), - "scope": scopes, - } - ) + scopes = urllib.parse.quote(scopes) + params = { + "client_id": wrap_with_braces("client_id", 3), + "redirect_uri": urlencode_field("redirect_uri"), + "response_type": "code", + "state": urlencode_field("state"), + "scope": "{" + scopes + "}", + } + + query = "&".join([f"{k}={v}" for k, v in params.items()]) url = urllib.parse.urlunsplit( [ "https", # scheme @@ -57,15 +58,14 @@ def build_auth_url_template() -> str: def build_access_token_url_template() -> str: - query = urllib.parse.urlencode( - { - "grant_type": "authorization_code", - "code": urlencode_field("code"), - "client_id": wrap_with_braces("client_id", 3), - "client_secret": wrap_with_braces("client_secret", 3), - "redirect_uri": urlencode_field("redirect_uri"), - } - ) + params = { + "grant_type": "authorization_code", + "code": urlencode_field("code"), + "client_id": wrap_with_braces("client_id", 3), + "client_secret": wrap_with_braces("client_secret", 3), + "redirect_uri": urlencode_field("redirect_uri"), + } + query = "&".join([f"{k}={v}" for k, v in params.items()]) url = urllib.parse.urlunsplit( [ "https", # scheme diff --git a/source-linkedin-pages/source_linkedin_pages/spec.json b/source-linkedin-pages/source_linkedin_pages/spec.json index 1ed45f0913..92e4335c39 100644 --- a/source-linkedin-pages/source_linkedin_pages/spec.json +++ b/source-linkedin-pages/source_linkedin_pages/spec.json @@ -1,5 +1,5 @@ { - "documentationUrl": "https://docs.estuary.dev/reference/Connectors/capture-connectors/linkedin-pages/", + "documentationUrl": "https://go.estuary.dev/source-linkedin-pages", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Linkedin Pages Spec", @@ -50,7 +50,7 @@ "refresh_token": { "type": "string", "title": "Refresh token", - "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://docs.estuary.dev/reference/Connectors/capture-connectors/linkedin-pages/.", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://go.estuary.dev/source-linkedin-pages.", "airbyte_secret": true } } @@ -69,7 +69,7 @@ "access_token": { "type": "string", "title": "Access token", - "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://docs.estuary.dev/reference/Connectors/capture-connectors/linkedin-pages/", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://go.estuary.dev/source-linkedin-pages", "airbyte_secret": true } } diff --git a/source-linkedin-pages/tests/snapshots/spec.stdout.json b/source-linkedin-pages/tests/snapshots/spec.stdout.json index 2ab4359d50..5ef4bed453 100644 --- a/source-linkedin-pages/tests/snapshots/spec.stdout.json +++ b/source-linkedin-pages/tests/snapshots/spec.stdout.json @@ -50,7 +50,7 @@ "refresh_token": { "type": "string", "title": "Refresh token", - "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://.com/integrations/sources/linkedin-pages/", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://go.estuary.dev/source-linkedin-pages.", "airbyte_secret": true } } @@ -69,7 +69,7 @@ "access_token": { "type": "string", "title": "Access token", - "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://docs.estuary.dev/reference/Connectors/capture-connectors/linkedin-pages/", + "description": "The token value generated using the LinkedIn Developers OAuth Token Tools. See the docs to obtain yours: https://go.estuary.dev/source-linkedin-pages", "airbyte_secret": true } } @@ -116,11 +116,11 @@ ], "additionalProperties": false }, - "documentationUrl": "https://docs.estuary.dev/reference/Connectors/capture-connectors/linkedin-pages/", + "documentationUrl": "https://go.estuary.dev/source-linkedin-pages", "oauth2": { "provider": "linkedin", - "authUrlTemplate": "https://www.linkedin.com/oauth/v2/authorization?client_id=%7B%7B%7Bclient_id%7D%7D%7D&redirect_uri=%7B%7B%23urlencode%7D%7D%7B%7B%7Bredirect_uri%7D%7D%7D%7B%7B%2Furlencode%7D%7D&response_type=code&state=%7B%7B%23urlencode%7D%7D%7B%7B%7Bstate%7D%7D%7D%7B%7B%2Furlencode%7D%7D&scope=r_organization_followers+r_organization_social+rw_organization_admin+r_organization_social_feed+w_member_social+w_organization_social+r_basicprofile+w_organization_social_feed+w_member_social_feed+r_1st_connections_size", - "accessTokenUrlTemplate": "https://www.linkedin.com/oauth/v2/accessToken?grant_type=authorization_code&code=%7B%7B%23urlencode%7D%7D%7B%7B%7Bcode%7D%7D%7D%7B%7B%2Furlencode%7D%7D&client_id=%7B%7B%7Bclient_id%7D%7D%7D&client_secret=%7B%7B%7Bclient_secret%7D%7D%7D&redirect_uri=%7B%7B%23urlencode%7D%7D%7B%7B%7Bredirect_uri%7D%7D%7D%7B%7B%2Furlencode%7D%7D", + "authUrlTemplate": "https://www.linkedin.com/oauth/v2/authorization?client_id={{{client_id}}}&redirect_uri={{#urlencode}}{{{redirect_uri}}}{{/urlencode}}&response_type=code&state={{#urlencode}}{{{state}}}{{/urlencode}}&scope={r_organization_followers%20r_organization_social%20rw_organization_admin%20r_organization_social_feed%20w_member_social%20w_organization_social%20r_basicprofile%20w_organization_social_feed%20w_member_social_feed%20r_1st_connections_size}", + "accessTokenUrlTemplate": "https://www.linkedin.com/oauth/v2/accessToken?grant_type=authorization_code&code={{#urlencode}}{{{code}}}{{/urlencode}}&client_id={{{client_id}}}&client_secret={{{client_secret}}}&redirect_uri={{#urlencode}}{{{redirect_uri}}}{{/urlencode}}", "accessTokenResponseMap": { "access_token": "/access_token", "refresh_token": "/refresh_token" From af61f15bff8bf4f4fe49415876f4fdb31cc7347b Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Thu, 4 Apr 2024 17:19:40 -0500 Subject: [PATCH 91/96] source-mysql: Repro test case for MySQL enum DDL bug There's a bug where when we process a DDL query involving an enum column, the parsed column type contains enum options wrapped in single quotes for some inexplicable reason, and so our metadata ends up being incorrect (compared to our own discovery schema generation and backfill behavior, as well as any sane understanding of the original DDL query's intent). It's bizarre that the Vitess query parser is doing this in the first place, but short of going in and modifying the parser logic the fix is obviously that we'll have to strip off the single quotes from the enum values in such cases. But before doing that, here's a test case which demonstrates the issue. --- .../TestAlterTable_AddEnumColumn-discover1 | 130 ++++++++++++++++ .../TestAlterTable_AddEnumColumn-discover2 | 143 ++++++++++++++++++ .../TestAlterTable_AddEnumColumn-init | 10 ++ .../TestAlterTable_AddEnumColumn-modified | 11 ++ .../TestAlterTable_AddEnumColumn-rebackfilled | 13 ++ source-mysql/main_test.go | 24 +++ 6 files changed, 331 insertions(+) create mode 100644 source-mysql/.snapshots/TestAlterTable_AddEnumColumn-discover1 create mode 100644 source-mysql/.snapshots/TestAlterTable_AddEnumColumn-discover2 create mode 100644 source-mysql/.snapshots/TestAlterTable_AddEnumColumn-init create mode 100644 source-mysql/.snapshots/TestAlterTable_AddEnumColumn-modified create mode 100644 source-mysql/.snapshots/TestAlterTable_AddEnumColumn-rebackfilled diff --git a/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-discover1 b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-discover1 new file mode 100644 index 0000000000..0f90c3a4b1 --- /dev/null +++ b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-discover1 @@ -0,0 +1,130 @@ +Binding 0: +{ + "recommended_name": "test_altertable_addenumcolumn_30213486", + "resource_config_json": { + "namespace": "test", + "stream": "AlterTable_AddEnumColumn_30213486" + }, + "document_schema_json": { + "$defs": { + "TestAlterTable_AddEnumColumn_30213486": { + "type": "object", + "required": [ + "id" + ], + "$anchor": "TestAlterTable_AddEnumColumn_30213486", + "properties": { + "data": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": "integer" + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, + "required": [ + "_meta" + ], + "properties": { + "_meta": { + "type": "object", + "required": [ + "op", + "source" + ], + "properties": { + "before": { + "$ref": "#TestAlterTable_AddEnumColumn_30213486", + "description": "Record state immediately before this change was applied.", + "reduce": { + "strategy": "firstWriteWins" + } + }, + "op": { + "enum": [ + "c", + "d", + "u" + ], + "description": "Change operation type: 'c' Create/Insert, 'u' Update, 'd' Delete." + }, + "source": { + "$id": "https://github.com/estuary/connectors/source-mysql/mysql-source-info", + "properties": { + "ts_ms": { + "type": "integer", + "description": "Unix timestamp (in millis) at which this event was recorded by the database." + }, + "schema": { + "type": "string", + "description": "Database schema (namespace) of the event." + }, + "snapshot": { + "type": "boolean", + "description": "Snapshot is true if the record was produced from an initial table backfill and unset if produced from the replication log." + }, + "table": { + "type": "string", + "description": "Database table of the event." + }, + "cursor": { + "type": "string", + "description": "Cursor value representing the current position in the binlog." + }, + "txid": { + "type": "string", + "description": "The global transaction identifier associated with a change by MySQL. Only set if GTIDs are enabled." + } + }, + "type": "object", + "required": [ + "schema", + "table", + "cursor" + ] + } + }, + "reduce": { + "strategy": "merge" + } + } + } + }, + { + "$ref": "#TestAlterTable_AddEnumColumn_30213486" + } + ] + }, + "key": [ + "/id" + ] + } + diff --git a/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-discover2 b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-discover2 new file mode 100644 index 0000000000..908bcb9b2a --- /dev/null +++ b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-discover2 @@ -0,0 +1,143 @@ +Binding 0: +{ + "recommended_name": "test_altertable_addenumcolumn_30213486", + "resource_config_json": { + "namespace": "test", + "stream": "AlterTable_AddEnumColumn_30213486" + }, + "document_schema_json": { + "$defs": { + "TestAlterTable_AddEnumColumn_30213486": { + "type": "object", + "required": [ + "id" + ], + "$anchor": "TestAlterTable_AddEnumColumn_30213486", + "properties": { + "data": { + "type": [ + "string", + "null" + ] + }, + "enumcol": { + "enum": [ + "", + "sm", + "med", + "lg", + null + ], + "type": [ + "string", + "null" + ] + }, + "id": { + "type": "integer" + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "_meta": { + "properties": { + "op": { + "const": "d" + } + } + } + } + }, + "then": { + "reduce": { + "delete": true, + "strategy": "merge" + } + }, + "else": { + "reduce": { + "strategy": "merge" + } + }, + "required": [ + "_meta" + ], + "properties": { + "_meta": { + "type": "object", + "required": [ + "op", + "source" + ], + "properties": { + "before": { + "$ref": "#TestAlterTable_AddEnumColumn_30213486", + "description": "Record state immediately before this change was applied.", + "reduce": { + "strategy": "firstWriteWins" + } + }, + "op": { + "enum": [ + "c", + "d", + "u" + ], + "description": "Change operation type: 'c' Create/Insert, 'u' Update, 'd' Delete." + }, + "source": { + "$id": "https://github.com/estuary/connectors/source-mysql/mysql-source-info", + "properties": { + "ts_ms": { + "type": "integer", + "description": "Unix timestamp (in millis) at which this event was recorded by the database." + }, + "schema": { + "type": "string", + "description": "Database schema (namespace) of the event." + }, + "snapshot": { + "type": "boolean", + "description": "Snapshot is true if the record was produced from an initial table backfill and unset if produced from the replication log." + }, + "table": { + "type": "string", + "description": "Database table of the event." + }, + "cursor": { + "type": "string", + "description": "Cursor value representing the current position in the binlog." + }, + "txid": { + "type": "string", + "description": "The global transaction identifier associated with a change by MySQL. Only set if GTIDs are enabled." + } + }, + "type": "object", + "required": [ + "schema", + "table", + "cursor" + ] + } + }, + "reduce": { + "strategy": "merge" + } + } + } + }, + { + "$ref": "#TestAlterTable_AddEnumColumn_30213486" + } + ] + }, + "key": [ + "/id" + ] + } + diff --git a/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-init b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-init new file mode 100644 index 0000000000..058f5097d1 --- /dev/null +++ b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-init @@ -0,0 +1,10 @@ +# ================================ +# Collection "acmeCo/test/test_altertable_addenumcolumn_30213486": 2 Documents +# ================================ +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"AlterTable_AddEnumColumn_30213486","cursor":"backfill:0"}},"data":"aaa","id":1} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"AlterTable_AddEnumColumn_30213486","cursor":"backfill:1"}},"data":"bbb","id":2} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test%2FAlterTable_AddEnumColumn_30213486":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data"],"types":{"data":"text","id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} + diff --git a/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-modified b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-modified new file mode 100644 index 0000000000..762162d0f8 --- /dev/null +++ b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-modified @@ -0,0 +1,11 @@ +# ================================ +# Collection "acmeCo/test/test_altertable_addenumcolumn_30213486": 3 Documents +# ================================ +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddEnumColumn_30213486","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"eee","enumcol":"'med'","id":3} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddEnumColumn_30213486","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"fff","enumcol":"'lg'","id":4} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddEnumColumn_30213486","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ggg","enumcol":"'sm'","id":5} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test%2FAlterTable_AddEnumColumn_30213486":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","enumcol"],"types":{"data":"text","enumcol":{"enum":["","'sm'","'med'","'lg'"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} + diff --git a/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-rebackfilled b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-rebackfilled new file mode 100644 index 0000000000..b177e7e663 --- /dev/null +++ b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-rebackfilled @@ -0,0 +1,13 @@ +# ================================ +# Collection "acmeCo/test/test_altertable_addenumcolumn_30213486": 5 Documents +# ================================ +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"AlterTable_AddEnumColumn_30213486","cursor":"backfill:0"}},"data":"aaa","enumcol":null,"id":1} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"AlterTable_AddEnumColumn_30213486","cursor":"backfill:1"}},"data":"bbb","enumcol":null,"id":2} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"AlterTable_AddEnumColumn_30213486","cursor":"backfill:2"}},"data":"eee","enumcol":"med","id":3} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"AlterTable_AddEnumColumn_30213486","cursor":"backfill:3"}},"data":"fff","enumcol":"lg","id":4} +{"_meta":{"op":"c","source":{"schema":"test","snapshot":true,"table":"AlterTable_AddEnumColumn_30213486","cursor":"backfill:4"}},"data":"ggg","enumcol":"sm","id":5} +# ================================ +# Final State Checkpoint +# ================================ +{"bindingStateV1":{"test%2FAlterTable_AddEnumColumn_30213486":{"backfilled":5,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","enumcol"],"types":{"data":"text","enumcol":{"enum":["","sm","med","lg"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} + diff --git a/source-mysql/main_test.go b/source-mysql/main_test.go index fcb1770633..34536d7070 100644 --- a/source-mysql/main_test.go +++ b/source-mysql/main_test.go @@ -435,6 +435,30 @@ func TestAlterTable_DropColumn(t *testing.T) { t.Run("restart", func(t *testing.T) { tests.VerifiedCapture(ctx, t, cs) }) } +func TestAlterTable_AddEnumColumn(t *testing.T) { + var tb, ctx = mysqlTestBackend(t), context.Background() + var uniqueID = "30213486" + var table = tb.CreateTable(ctx, t, uniqueID, "(id INTEGER PRIMARY KEY, data TEXT)") + tb.Insert(ctx, t, table, [][]interface{}{{1, "aaa"}, {2, "bbb"}}) + + t.Run("discover1", func(t *testing.T) { tb.CaptureSpec(ctx, t).VerifyDiscover(ctx, t, regexp.MustCompile(uniqueID)) }) + + var cs = tb.CaptureSpec(ctx, t, regexp.MustCompile(uniqueID)) + t.Run("init", func(t *testing.T) { tests.VerifiedCapture(ctx, t, cs) }) + + tb.Query(ctx, t, fmt.Sprintf("ALTER TABLE %s ADD COLUMN enumcol ENUM('sm', 'med','lg');", table)) + tb.Insert(ctx, t, table, [][]interface{}{ + {3, "eee", "med"}, + {4, "fff", "lg"}, + {5, "ggg", "sm"}, + }) + t.Run("modified", func(t *testing.T) { tests.VerifiedCapture(ctx, t, cs) }) + + t.Run("discover2", func(t *testing.T) { tb.CaptureSpec(ctx, t).VerifyDiscover(ctx, t, regexp.MustCompile(uniqueID)) }) + cs = tb.CaptureSpec(ctx, t, regexp.MustCompile(uniqueID)) + t.Run("rebackfilled", func(t *testing.T) { tests.VerifiedCapture(ctx, t, cs) }) +} + // TestBinlogExpirySanityCheck verifies that the "dangerously short binlog expiry" // sanity check is working as intended. func TestBinlogExpirySanityCheck(t *testing.T) { From 17529caaa4c50f0422df02c8dc824a418c6b6c5f Mon Sep 17 00:00:00 2001 From: Will Donnelly Date: Thu, 4 Apr 2024 17:48:01 -0500 Subject: [PATCH 92/96] source-mysql: Strip single-quoted EnumValues in parsed DDL For some reason the Vitess SQL parser tokenizes single-quoted strings into the underlying string contents, but then when it comes time to construct the parsed AST that string gets quoted and escaped back up with an explicit call to `encodeSQLString()` I have no idea why it does that, but it's clearly a deliberate choice. So okay, now we'll have some logic to undo that again and get us back down to the raw strings. This is already something we have to do when processing the list of enum values in discovery, so really we just had to factor out the logic to unquote a single such string and reuse it here. --- ...stAlterTable_AddColumnSetEnum-enum-restart | 6 ++--- ...estAlterTable_AddColumnSetEnum-enum-stream | 6 ++--- ...estAlterTable_AddColumnSetEnum-set-restart | 6 ++--- ...TestAlterTable_AddColumnSetEnum-set-stream | 6 ++--- .../TestAlterTable_AddEnumColumn-modified | 8 +++---- source-mysql/discovery.go | 19 +++++++++++---- source-mysql/replication.go | 23 +++++++++++++++++-- 7 files changed, 51 insertions(+), 23 deletions(-) diff --git a/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-enum-restart b/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-enum-restart index 2b614365ad..9a8012c9dc 100644 --- a/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-enum-restart +++ b/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-enum-restart @@ -1,10 +1,10 @@ # ================================ # Collection "acmeCo/test/test_altertable_addcolumnsetenum_enum_76927424": 2 Documents # ================================ -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_enum_76927424","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"eee","enumCol":"'someValue'","id":5} -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_enum_76927424","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"fff","enumCol":"'someValue'","id":6} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_enum_76927424","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"eee","enumCol":"someValue","id":5} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_enum_76927424","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"fff","enumCol":"someValue","id":6} # ================================ # Final State Checkpoint # ================================ -{"bindingStateV1":{"test%2FAlterTable_AddColumnSetEnum_enum_76927424":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","enumCol"],"types":{"data":"text","enumCol":{"enum":["","'someValue'","'anotherValue'"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} +{"bindingStateV1":{"test%2FAlterTable_AddColumnSetEnum_enum_76927424":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","enumCol"],"types":{"data":"text","enumCol":{"enum":["","someValue","anotherValue"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} diff --git a/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-enum-stream b/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-enum-stream index 5dbb5eb7a8..5c76ab64c8 100644 --- a/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-enum-stream +++ b/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-enum-stream @@ -1,10 +1,10 @@ # ================================ # Collection "acmeCo/test/test_altertable_addcolumnsetenum_enum_76927424": 2 Documents # ================================ -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_enum_76927424","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ccc","enumCol":"'anotherValue'","id":3} -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_enum_76927424","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ddd","enumCol":"'someValue'","id":4} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_enum_76927424","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ccc","enumCol":"anotherValue","id":3} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_enum_76927424","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ddd","enumCol":"someValue","id":4} # ================================ # Final State Checkpoint # ================================ -{"bindingStateV1":{"test%2FAlterTable_AddColumnSetEnum_enum_76927424":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","enumCol"],"types":{"data":"text","enumCol":{"enum":["","'someValue'","'anotherValue'"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} +{"bindingStateV1":{"test%2FAlterTable_AddColumnSetEnum_enum_76927424":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","enumCol"],"types":{"data":"text","enumCol":{"enum":["","someValue","anotherValue"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} diff --git a/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-set-restart b/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-set-restart index dba7b3d07c..74e468a7b3 100644 --- a/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-set-restart +++ b/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-set-restart @@ -1,10 +1,10 @@ # ================================ # Collection "acmeCo/test/test_altertable_addcolumnsetenum_set_14622082": 2 Documents # ================================ -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_set_14622082","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"eee","id":5,"setCol":"'a','c'"} -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_set_14622082","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"fff","id":6,"setCol":"'b','c'"} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_set_14622082","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"eee","id":5,"setCol":"a,c"} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_set_14622082","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"fff","id":6,"setCol":"b,c"} # ================================ # Final State Checkpoint # ================================ -{"bindingStateV1":{"test%2FAlterTable_AddColumnSetEnum_set_14622082":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","setCol"],"types":{"data":"text","id":"int","setCol":{"enum":["'a'","'b'","'c'"],"type":"set"}}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} +{"bindingStateV1":{"test%2FAlterTable_AddColumnSetEnum_set_14622082":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","setCol"],"types":{"data":"text","id":"int","setCol":{"enum":["a","b","c"],"type":"set"}}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} diff --git a/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-set-stream b/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-set-stream index 07d889885e..adb115b8c2 100644 --- a/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-set-stream +++ b/source-mysql/.snapshots/TestAlterTable_AddColumnSetEnum-set-stream @@ -1,10 +1,10 @@ # ================================ # Collection "acmeCo/test/test_altertable_addcolumnsetenum_set_14622082": 2 Documents # ================================ -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_set_14622082","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ccc","id":3,"setCol":"'a','b'"} -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_set_14622082","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ddd","id":4,"setCol":"'b','c'"} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_set_14622082","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ccc","id":3,"setCol":"a,b"} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddColumnSetEnum_set_14622082","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ddd","id":4,"setCol":"b,c"} # ================================ # Final State Checkpoint # ================================ -{"bindingStateV1":{"test%2FAlterTable_AddColumnSetEnum_set_14622082":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","setCol"],"types":{"data":"text","id":"int","setCol":{"enum":["'a'","'b'","'c'"],"type":"set"}}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} +{"bindingStateV1":{"test%2FAlterTable_AddColumnSetEnum_set_14622082":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","setCol"],"types":{"data":"text","id":"int","setCol":{"enum":["a","b","c"],"type":"set"}}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} diff --git a/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-modified b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-modified index 762162d0f8..cf860d8c5a 100644 --- a/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-modified +++ b/source-mysql/.snapshots/TestAlterTable_AddEnumColumn-modified @@ -1,11 +1,11 @@ # ================================ # Collection "acmeCo/test/test_altertable_addenumcolumn_30213486": 3 Documents # ================================ -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddEnumColumn_30213486","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"eee","enumcol":"'med'","id":3} -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddEnumColumn_30213486","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"fff","enumcol":"'lg'","id":4} -{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddEnumColumn_30213486","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ggg","enumcol":"'sm'","id":5} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddEnumColumn_30213486","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"eee","enumcol":"med","id":3} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddEnumColumn_30213486","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"fff","enumcol":"lg","id":4} +{"_meta":{"op":"c","source":{"ts_ms":1111111111111,"schema":"test","table":"AlterTable_AddEnumColumn_30213486","cursor":"binlog.000123:56789:123","txid":"11111111-1111-1111-1111-111111111111:111"}},"data":"ggg","enumcol":"sm","id":5} # ================================ # Final State Checkpoint # ================================ -{"bindingStateV1":{"test%2FAlterTable_AddEnumColumn_30213486":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","enumcol"],"types":{"data":"text","enumcol":{"enum":["","'sm'","'med'","'lg'"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} +{"bindingStateV1":{"test%2FAlterTable_AddEnumColumn_30213486":{"backfilled":2,"key_columns":["id"],"metadata":{"schema":{"columns":["id","data","enumcol"],"types":{"data":"text","enumcol":{"enum":["","sm","med","lg"],"type":"enum"},"id":"int"}}},"mode":"Active"}},"cursor":"binlog.000123:56789"} diff --git a/source-mysql/discovery.go b/source-mysql/discovery.go index 8fd678d07d..94227bd8f3 100644 --- a/source-mysql/discovery.go +++ b/source-mysql/discovery.go @@ -463,7 +463,7 @@ var enumValuesRegexp = regexp.MustCompile(`'((?:''|\\.|[^'])+)'(?:,|$)`) // enumValueReplacements contains the complete list of MySQL string escapes from // https://dev.mysql.com/doc/refman/8.0/en/string-literals.html#character-escape-sequences -// plus the `”` repeated-single-quote mechanism. +// plus the `'​'` repeated-single-quote mechanism. var enumValueReplacements = map[string]string{ `''`: "'", `\0`: "\x00", @@ -492,13 +492,22 @@ func parseEnumValues(details string) []string { // and take submatch #1 which is the body of each string. var opts []string for _, match := range enumValuesRegexp.FindAllStringSubmatch(details, -1) { - var opt = match[1] + opts = append(opts, decodeMySQLString(match[1])) + } + return opts +} + +// decodeStringMySQL decodes a MySQL-format single-quoted string (including +// possible backslash escapes) and returns it in unquoted, unescaped form. +func decodeMySQLString(qstr string) string { + if strings.HasPrefix(qstr, "'") && strings.HasSuffix(qstr, "'") { + qstr = strings.TrimPrefix(qstr, "'") + qstr = strings.TrimSuffix(qstr, "'") for old, new := range enumValueReplacements { - opt = strings.ReplaceAll(opt, old, new) + qstr = strings.ReplaceAll(qstr, old, new) } - opts = append(opts, opt) } - return opts + return qstr } const queryDiscoverPrimaryKeys = ` diff --git a/source-mysql/replication.go b/source-mysql/replication.go index 17bdab7494..ca87e89bc8 100644 --- a/source-mysql/replication.go +++ b/source-mysql/replication.go @@ -682,13 +682,32 @@ func (rs *mysqlReplicationStream) handleAlterTable(ctx context.Context, stmt *sq func translateDataType(t sqlparser.ColumnType) any { var typeName = strings.ToLower(t.Type) if typeName == "enum" { - return &mysqlColumnType{Type: typeName, EnumValues: append([]string{""}, t.EnumValues...)} + return &mysqlColumnType{Type: typeName, EnumValues: append([]string{""}, unquoteEnumValues(t.EnumValues)...)} } else if typeName == "set" { - return &mysqlColumnType{Type: typeName, EnumValues: t.EnumValues} + return &mysqlColumnType{Type: typeName, EnumValues: unquoteEnumValues(t.EnumValues)} } return typeName } +// unquoteEnumValues applies MySQL single-quote-unescaping to a list of single-quoted +// escaped string values such as the EnumValues list returned by the Vitess SQL Parser +// package when parsing a DDL query involving enum/set values. +// +// The single-quote wrapping and escaping of these strings is clearly deliberate, as +// under the hood the package actually tokenizes the strings to raw values and then +// explicitly calls `encodeSQLString()` to re-wrap them when building the AST. The +// actual reason for doing this is unknown however, and it makes very little sense. +// +// So whatever, here's a helper function to undo that escaping and get back down to +// the raw strings again. +func unquoteEnumValues(values []string) []string { + var unquoted []string + for _, qval := range values { + unquoted = append(unquoted, decodeMySQLString(qval)) + } + return unquoted +} + func findStr(needle string, cols []string) int { for idx, val := range cols { if val == needle { From 671fd46681e64aab99083c26d465986c1db6d7f9 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 5 Apr 2024 17:26:17 +0100 Subject: [PATCH 93/96] materialize-snowflake: round-robin try pipes --- materialize-snowflake/snowflake.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/materialize-snowflake/snowflake.go b/materialize-snowflake/snowflake.go index 06e33900f7..63e532bffa 100644 --- a/materialize-snowflake/snowflake.go +++ b/materialize-snowflake/snowflake.go @@ -620,10 +620,16 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error // If we see no results from the REST API for `maxTries` iterations, then we // fallback to asking the `COPY_HISTORY` table. We allow up to 5 minutes for results to show up in the REST API. - var maxTries = 60 - var retryDelaySeconds = 5 * time.Second - for pipeName, pipe := range pipes { - for tries := 0; tries < maxTries; tries++ { + var maxTries = 20 + var retryDelaySeconds = 3 * time.Second + for tries := 0; tries < maxTries; tries++ { + for pipeName, pipe := range pipes { + // if the pipe has no files to begin with, just skip it + // we might have processed all files of this pipe previously + if len(pipe.files) == 0 { + continue + } + // We first try to check the status of pipes using the REST API's insertReport // The REST API does not wake up the warehouse, hence our preference // however this API is not the most ergonomic and sometimes does not yield @@ -637,7 +643,6 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error // which is ahead, and some results have not been seen by us and will not be // seen with the given cursor. So we reset the cursor in this case and try again. if !report.CompleteResult { - tries-- d.pipeMarkers[pipeName] = "" } else { d.pipeMarkers[pipeName] = report.NextBeginMark @@ -699,7 +704,7 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error // All files have been processed for this pipe, we can skip to the next pipe d.deleteFiles(ctx, []string{pipe.dir}) - break + continue } // So long as we are able to get some results from the REST API, we do not @@ -718,7 +723,7 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error if len(pipe.files) == 0 { d.deleteFiles(ctx, []string{pipe.dir}) - break + continue } else { time.Sleep(retryDelaySeconds) } From 1316eb9fa7f8426b93a069928453c432967b8b14 Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Mon, 8 Apr 2024 14:50:35 +0100 Subject: [PATCH 94/96] source-mysql: ignore more complicated CREATE ... DEFINER ... VIEW qs --- source-mysql/replication.go | 3 ++- source-mysql/replication_test.go | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source-mysql/replication.go b/source-mysql/replication.go index ca87e89bc8..5daa2b6ad8 100644 --- a/source-mysql/replication.go +++ b/source-mysql/replication.go @@ -453,7 +453,8 @@ func decodeRow(streamID string, colNames []string, row []interface{}) (map[strin // TODO(johnny): SET STATEMENT is not safe in the general case, and we want to re-visit // by extracting and ignoring a SET STATEMENT stanza prior to parsing. var silentIgnoreQueriesRe = regexp.MustCompile(`(?i)^(BEGIN|# [^\n]*)$`) -var ignoreQueriesRe = regexp.MustCompile(`(?i)^(BEGIN|COMMIT|GRANT|REVOKE|CREATE USER|CREATE DEFINER|DROP USER|ALTER USER|DROP PROCEDURE|DROP FUNCTION|DROP TRIGGER|SET STATEMENT|# |/\*|-- )`) +var createDefinerRegex = `CREATE\s*(OR REPLACE){0,1}\s*(ALGORITHM\s*=\s*[^ ]+)*\s*DEFINER` +var ignoreQueriesRe = regexp.MustCompile(`(?i)^(BEGIN|COMMIT|GRANT|REVOKE|CREATE USER|` + createDefinerRegex + `|DROP USER|ALTER USER|DROP PROCEDURE|DROP FUNCTION|DROP TRIGGER|SET STATEMENT|# |/\*|-- )`) func (rs *mysqlReplicationStream) handleQuery(ctx context.Context, schema, query string) error { // There are basically three types of query events we might receive: diff --git a/source-mysql/replication_test.go b/source-mysql/replication_test.go index 716ca5f5ea..bb31a93e63 100644 --- a/source-mysql/replication_test.go +++ b/source-mysql/replication_test.go @@ -8,6 +8,10 @@ func TestIgnoreQueries(t *testing.T) { `/* This is also a comment */`: true, `BEGIN`: true, `COMMIT`: true, + `CREATE DEFINER`: true, + `CREATE OR REPLACE DEFINER`: true, + `CREATE OR REPLACE ALGORITHM=UNDEFINED DEFINER`: true, + `CREATE ALGORITHM = TEMPTABLE DEFINER`: true, `CREATE USER IF NOT EXISTS flow_capture IDENTIFIED BY 'secret1234'`: true, From 5ff81a5bb367f548e9d59cce862c9c9fa5da66d7 Mon Sep 17 00:00:00 2001 From: Jon Wihl Date: Mon, 8 Apr 2024 12:27:36 -0400 Subject: [PATCH 95/96] Added Jira Variant --- source-http-ingest/VARIANTS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source-http-ingest/VARIANTS b/source-http-ingest/VARIANTS index 6a7f471c77..e70520ca22 100644 --- a/source-http-ingest/VARIANTS +++ b/source-http-ingest/VARIANTS @@ -1,2 +1,3 @@ source-datadog-ingest -source-intercom-ingest \ No newline at end of file +source-intercom-ingest +source-jira-ingest \ No newline at end of file From edfda36f09f1372b790d79c9591a19d6c0cad01f Mon Sep 17 00:00:00 2001 From: Will Baker Date: Wed, 3 Apr 2024 18:00:48 -0400 Subject: [PATCH 96/96] materializations: don't try to drop nullability if the projection has a default value In e852e80, materializations were made aware of the possibility for default values in the materialized columns, and that we should not try to drop "not null" constraints on columns like this, under the assumption that the destination system is fine with us sending a "null" since it will automatically be populated (in a sense) with that default value. This didn't take into consideration the fact that Flow projections can also have default values, and in this case we will never see a "null" for these fields from the Flow runtime, and instead will always see that default value. So here too we do not need to drop nullability constraints - the materialized columns can stay "not null" and as long as there is that default value annotation things will work fine. This is particularly important for materializing nullable collection keys with default values. This is allowed (see 5d53313 as well), and some systems like postgres consider primary key fields non-nullable. Prior to this commit, the connector would try to drop a "not null" constraint on a primary key column that was created from a projection with a default value if the materialized column didn't have a default value. We don't currently create columns with default values based on projections (though maybe we will, later) so this would cause the materialization to fail the next time it tries to do an `Apply`. But now, the materialization will not try to drop a "not null" on a materialized key field as long as the projection has a default value. --- materialize-boilerplate/apply.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/materialize-boilerplate/apply.go b/materialize-boilerplate/apply.go index 15553203e7..5e4815f0f0 100644 --- a/materialize-boilerplate/apply.go +++ b/materialize-boilerplate/apply.go @@ -178,10 +178,12 @@ func ApplyChanges(ctx context.Context, req *pm.Request_Apply, applier Applier, i } newRequired := projection.Inference.Exists == pf.Inference_MUST && !slices.Contains(projection.Inference.Types, pf.JsonTypeNull) - if !existingField.Nullable && !newRequired && !existingField.HasDefault { - // The existing field is not nullable and does not have a default value, but - // the proposed projection for the field is nullable. The existing field - // will need to be modified to be made nullable. + newlyNullable := !existingField.Nullable && !newRequired + projectionHasDefault := projection.Inference.DefaultJson != nil + if newlyNullable && !existingField.HasDefault && !projectionHasDefault { + // The field has newly been made nullable and neither the existing field nor + // the projection has a default value. The existing field will need to be + // modified to be made nullable since it may need to hold null values now. params.NewlyNullableFields = append(params.NewlyNullableFields, existingField) } } else {