From 8feeb8f8ee1abc538ab4199c5d9e2d3fc3099ce8 Mon Sep 17 00:00:00 2001 From: amigin Date: Fri, 22 Nov 2024 11:25:29 +0200 Subject: [PATCH] New Persist Format --- src/app/app_ctx.rs | 28 +- .../read/get_highest_row_and_below.rs | 2 +- src/db_operations/read/get_rows_as_vec.rs | 4 +- src/db_operations/read/rows/get_all.rs | 2 +- .../read/rows/get_all_by_partition_key.rs | 2 +- .../read/rows/get_all_by_row_key.rs | 2 +- src/db_operations/read/rows/get_single.rs | 2 +- .../get_single_partition_multiple_rows.rs | 2 +- .../update_partition_expiration_time.rs | 18 +- .../update_rows_expiration_time.rs | 17 +- .../update_statistics/update_statistics.rs | 11 +- src/db_operations/write/bulk_delete.rs | 17 +- .../write/bulk_insert_or_update.rs | 7 +- .../write/clean_partition_and_bulk_insert.rs | 2 +- src/db_operations/write/clean_table.rs | 2 +- .../write/clean_table_and_bulk_insert.rs | 2 +- src/db_operations/write/delete_partitions.rs | 8 +- src/db_operations/write/delete_row.rs | 7 +- src/db_operations/write/insert.rs | 7 +- src/db_operations/write/insert_or_replace.rs | 7 +- src/db_operations/write/replace.rs | 7 +- src/db_operations/write/table.rs | 36 +-- src/db_sync/states/sync_table_data.rs | 6 +- src/db_sync/sync_event.rs | 14 +- .../get_changes_action.rs | 3 +- .../controllers/status_controller/models.rs | 2 +- .../non_initialized/model.rs | 18 +- src/main.rs | 14 +- src/operations/build_db_snapshot_as_zip.rs | 2 +- .../init/entities_from_sqlite_reader.rs | 55 ++++ .../init/from_other_instance/load_rows.rs | 41 +++ .../init/from_other_instance/load_tables.rs | 57 ++++ .../init}/from_other_instance/mod.rs | 2 - src/operations/init/init_state.rs | 31 ++ src/operations/init/load_tables.rs | 44 +++ src/operations/init/mod.rs | 20 ++ src/operations/init/scripts.rs | 53 ++++ src/operations/mod.rs | 5 +- src/operations/persist.rs | 84 ------ src/operations/persist/mod.rs | 12 + src/operations/persist/persist.rs | 53 ++++ src/operations/persist/save_partition.rs | 31 ++ src/operations/persist/save_rows.rs | 69 +++++ src/operations/persist/save_table.rs | 15 + .../persist/save_table_attributes.rs | 15 + src/operations/persist/scripts.rs | 88 ++++++ src/persist/mod.rs | 5 - src/persist/persist_markers.rs | 32 -- src/persist/persist_markers_by_table.rs | 181 ------------ src/persist_io/azure/attempt_handling.rs | 26 -- src/persist_io/azure/create_table.rs | 19 -- src/persist_io/azure/delete_table_file.rs | 28 -- src/persist_io/azure/delete_table_folder.rs | 18 -- src/persist_io/azure/get_list_of_files.rs | 65 ----- src/persist_io/azure/get_list_of_tables.rs | 48 --- src/persist_io/azure/load_table_file.rs | 43 --- src/persist_io/azure/mod.rs | 15 - src/persist_io/azure/save_attributes.rs | 34 --- src/persist_io/azure/save_table_file.rs | 28 -- src/persist_io/mod.rs | 8 - src/persist_io/persist_io_operations.rs | 229 --------------- src/persist_io/sqlite/init_container.rs | 74 ----- .../sqlite/init_new_instance_table.rs | 43 --- src/persist_io/sqlite/mod.rs | 9 - src/persist_io/table_file.rs | 81 ------ src/persist_markers/mod.rs | 15 + src/persist_markers/persist_by_table_item.rs | 170 +++++++++++ src/persist_markers/persist_markers.rs | 274 ++++++++++++++++++ src/persist_markers/persist_markers_inner.rs | 126 ++++++++ src/persist_markers/persist_metrics.rs | 23 ++ .../persist_partition_marker.rs} | 96 +++--- src/persist_markers/persist_row_marker.rs | 26 ++ src/persist_markers/persist_task.rs | 21 ++ .../blob_content_cache/blob_content_cache.rs | 140 --------- .../blob_content_cache/mod.rs | 4 - .../persisted_table_data.rs | 34 --- .../init_from_another_instance.rs | 67 ----- .../from_other_instance/load_rows.rs | 18 -- .../from_other_instance/load_tables.rs | 25 -- .../data_initializer/load_table_files.rs | 62 ---- .../data_initializer/load_tables.rs | 120 -------- .../data_initializer/load_tasks/init_state.rs | 112 ------- .../load_tasks/init_state_data.rs | 123 -------- .../load_tasks/init_state_snapshot.rs | 6 - .../load_tasks/load_table_task.rs | 120 -------- .../data_initializer/load_tasks/mod.rs | 9 - .../data_initializer/loaded_table_item.rs | 28 -- .../data_initializer/mod.rs | 10 - .../table_list_of_files_loader.rs | 11 - src/persist_operations/mod.rs | 4 - src/persist_operations/serializers/mod.rs | 3 - src/persist_operations/sync/create_table.rs | 17 -- .../sync/delete_partition.rs | 27 -- src/persist_operations/sync/delete_table.rs | 17 -- src/persist_operations/sync/mod.rs | 14 - src/persist_operations/sync/save_partition.rs | 51 ---- src/persist_operations/sync/save_table.rs | 55 ---- .../sync/save_table_attributes.rs | 17 -- .../sync/upload_partition.rs | 19 -- src/scripts/mod.rs | 2 + .../serializers/db_partition.rs | 2 +- src/scripts/serializers/mod.rs | 2 + .../serializers/table_attrs.rs | 0 src/settings_reader.rs | 38 +-- src/sqlite_repo/entity_dto.rs | 45 +++ src/sqlite_repo/{row_dto.rs => file_dto.rs} | 0 src/sqlite_repo/mod.rs | 8 +- src/sqlite_repo/sqlite_repo.rs | 99 ++++++- src/sqlite_repo/table_dto.rs | 28 -- src/sqlite_repo/table_metadata_dto.rs | 52 ++++ src/tcp/tcp_server_events.rs | 3 +- src/zip/db_zip_builder.rs | 4 +- test/test-case.json | 1 - test/test_case_2.json | 1 - 114 files changed, 1664 insertions(+), 2397 deletions(-) create mode 100644 src/operations/init/entities_from_sqlite_reader.rs create mode 100644 src/operations/init/from_other_instance/load_rows.rs create mode 100644 src/operations/init/from_other_instance/load_tables.rs rename src/{persist_operations/data_initializer => operations/init}/from_other_instance/mod.rs (52%) create mode 100644 src/operations/init/init_state.rs create mode 100644 src/operations/init/load_tables.rs create mode 100644 src/operations/init/mod.rs create mode 100644 src/operations/init/scripts.rs delete mode 100644 src/operations/persist.rs create mode 100644 src/operations/persist/mod.rs create mode 100644 src/operations/persist/persist.rs create mode 100644 src/operations/persist/save_partition.rs create mode 100644 src/operations/persist/save_rows.rs create mode 100644 src/operations/persist/save_table.rs create mode 100644 src/operations/persist/save_table_attributes.rs create mode 100644 src/operations/persist/scripts.rs delete mode 100644 src/persist/mod.rs delete mode 100644 src/persist/persist_markers.rs delete mode 100644 src/persist/persist_markers_by_table.rs delete mode 100644 src/persist_io/azure/attempt_handling.rs delete mode 100644 src/persist_io/azure/create_table.rs delete mode 100644 src/persist_io/azure/delete_table_file.rs delete mode 100644 src/persist_io/azure/delete_table_folder.rs delete mode 100644 src/persist_io/azure/get_list_of_files.rs delete mode 100644 src/persist_io/azure/get_list_of_tables.rs delete mode 100644 src/persist_io/azure/load_table_file.rs delete mode 100644 src/persist_io/azure/mod.rs delete mode 100644 src/persist_io/azure/save_attributes.rs delete mode 100644 src/persist_io/azure/save_table_file.rs delete mode 100644 src/persist_io/mod.rs delete mode 100644 src/persist_io/persist_io_operations.rs delete mode 100644 src/persist_io/sqlite/init_container.rs delete mode 100644 src/persist_io/sqlite/init_new_instance_table.rs delete mode 100644 src/persist_io/sqlite/mod.rs delete mode 100644 src/persist_io/table_file.rs create mode 100644 src/persist_markers/mod.rs create mode 100644 src/persist_markers/persist_by_table_item.rs create mode 100644 src/persist_markers/persist_markers.rs create mode 100644 src/persist_markers/persist_markers_inner.rs create mode 100644 src/persist_markers/persist_metrics.rs rename src/{persist/partition_persist_marker.rs => persist_markers/persist_partition_marker.rs} (71%) create mode 100644 src/persist_markers/persist_row_marker.rs create mode 100644 src/persist_markers/persist_task.rs delete mode 100644 src/persist_operations/blob_content_cache/blob_content_cache.rs delete mode 100644 src/persist_operations/blob_content_cache/mod.rs delete mode 100644 src/persist_operations/blob_content_cache/persisted_table_data.rs delete mode 100644 src/persist_operations/data_initializer/from_other_instance/init_from_another_instance.rs delete mode 100644 src/persist_operations/data_initializer/from_other_instance/load_rows.rs delete mode 100644 src/persist_operations/data_initializer/from_other_instance/load_tables.rs delete mode 100644 src/persist_operations/data_initializer/load_table_files.rs delete mode 100644 src/persist_operations/data_initializer/load_tables.rs delete mode 100644 src/persist_operations/data_initializer/load_tasks/init_state.rs delete mode 100644 src/persist_operations/data_initializer/load_tasks/init_state_data.rs delete mode 100644 src/persist_operations/data_initializer/load_tasks/init_state_snapshot.rs delete mode 100644 src/persist_operations/data_initializer/load_tasks/load_table_task.rs delete mode 100644 src/persist_operations/data_initializer/load_tasks/mod.rs delete mode 100644 src/persist_operations/data_initializer/loaded_table_item.rs delete mode 100644 src/persist_operations/data_initializer/mod.rs delete mode 100644 src/persist_operations/data_initializer/table_list_of_files_loader.rs delete mode 100644 src/persist_operations/mod.rs delete mode 100644 src/persist_operations/serializers/mod.rs delete mode 100644 src/persist_operations/sync/create_table.rs delete mode 100644 src/persist_operations/sync/delete_partition.rs delete mode 100644 src/persist_operations/sync/delete_table.rs delete mode 100644 src/persist_operations/sync/mod.rs delete mode 100644 src/persist_operations/sync/save_partition.rs delete mode 100644 src/persist_operations/sync/save_table.rs delete mode 100644 src/persist_operations/sync/save_table_attributes.rs delete mode 100644 src/persist_operations/sync/upload_partition.rs create mode 100644 src/scripts/mod.rs rename src/{persist_operations => scripts}/serializers/db_partition.rs (95%) create mode 100644 src/scripts/serializers/mod.rs rename src/{persist_operations => scripts}/serializers/table_attrs.rs (100%) create mode 100644 src/sqlite_repo/entity_dto.rs rename src/sqlite_repo/{row_dto.rs => file_dto.rs} (100%) delete mode 100644 src/sqlite_repo/table_dto.rs create mode 100644 src/sqlite_repo/table_metadata_dto.rs delete mode 100644 test/test-case.json delete mode 100644 test/test_case_2.json diff --git a/src/app/app_ctx.rs b/src/app/app_ctx.rs index c97c4bd..254c9a4 100644 --- a/src/app/app_ctx.rs +++ b/src/app/app_ctx.rs @@ -7,15 +7,9 @@ use my_no_sql_sdk::core::rust_extensions::{date_time::DateTimeAsMicroseconds, Ap use my_no_sql_server_core::DbInstance; use crate::{ - data_readers::DataReadersList, - db_operations::multipart::MultipartList, - db_transactions::ActiveTransactions, - persist::PersistMarkersByTable, - persist_io::PersistIoOperations, - persist_operations::{ - blob_content_cache::BlobContentCache, data_initializer::load_tasks::InitState, - }, - settings_reader::SettingsModel, + data_readers::DataReadersList, db_operations::multipart::MultipartList, + db_transactions::ActiveTransactions, operations::init::InitState, + persist_markers::PersistMarkers, settings_reader::SettingsModel, }; use super::{EventsSync, HttpWriters, PrometheusMetrics}; @@ -34,40 +28,40 @@ pub struct AppContext { pub active_transactions: ActiveTransactions, pub process_id: String, - pub blob_content_cache: BlobContentCache, pub data_readers: DataReadersList, pub multipart_list: MultipartList, - pub persist_io: PersistIoOperations, + //pub persist_io: PersistIoOperations, pub init_state: InitState, + pub repo: crate::sqlite_repo::SqlLiteRepo, + pub settings: Arc, pub sync: EventsSync, pub states: Arc, - pub persist_markers: PersistMarkersByTable, + pub persist_markers: PersistMarkers, pub http_writers: HttpWriters, persist_amount: AtomicUsize, } impl AppContext { - pub fn new(settings: Arc, persist_io: PersistIoOperations) -> Self { + pub async fn new(settings: Arc) -> Self { AppContext { - persist_markers: PersistMarkersByTable::new(), + persist_markers: PersistMarkers::new(), created: DateTimeAsMicroseconds::now(), - init_state: InitState::new(), db: DbInstance::new(), metrics: PrometheusMetrics::new(), active_transactions: ActiveTransactions::new(), process_id: uuid::Uuid::new_v4().to_string(), states: Arc::new(AppStates::create_un_initialized()), - blob_content_cache: BlobContentCache::new(), data_readers: DataReadersList::new(Duration::from_secs(30)), multipart_list: MultipartList::new(), - persist_io, + repo: settings.get_sqlite_repo().await, settings, persist_amount: AtomicUsize::new(0), sync: EventsSync::new(), http_writers: HttpWriters::new(), + init_state: InitState::new(), } } diff --git a/src/db_operations/read/get_highest_row_and_below.rs b/src/db_operations/read/get_highest_row_and_below.rs index 973cdb0..8ff2258 100644 --- a/src/db_operations/read/get_highest_row_and_below.rs +++ b/src/db_operations/read/get_highest_row_and_below.rs @@ -40,7 +40,7 @@ pub async fn get_highest_row_and_below( break; } } - update_statistics.update(db_partition, Some(db_row), now); + update_statistics.update(db_table_wrapper, db_partition, Some(db_row), now); json_array_writer.write(db_row.as_ref()); count += 1; diff --git a/src/db_operations/read/get_rows_as_vec.rs b/src/db_operations/read/get_rows_as_vec.rs index a7c7d37..dc385ad 100644 --- a/src/db_operations/read/get_rows_as_vec.rs +++ b/src/db_operations/read/get_rows_as_vec.rs @@ -64,7 +64,7 @@ pub async fn get_as_partition_key_and_row_key( let db_row = db_partition.get_row_and_clone(row_key)?; - update_statistics.update(db_partition, Some(&db_row), now.date_time); + update_statistics.update(table, db_partition, Some(&db_row), now.date_time); Some(db_row) } @@ -109,7 +109,7 @@ async fn get_as_row_key_only( let mut result = Vec::new(); for (db_partition, db_row) in read_access.get_by_row_key(row_key, skip, limit) { - update_statistics.update(db_partition, Some(db_row), now.date_time); + update_statistics.update(table, db_partition, Some(db_row), now.date_time); result.push(db_row.clone()); } diff --git a/src/db_operations/read/rows/get_all.rs b/src/db_operations/read/rows/get_all.rs index bbcfaf5..3731f1f 100644 --- a/src/db_operations/read/rows/get_all.rs +++ b/src/db_operations/read/rows/get_all.rs @@ -25,7 +25,7 @@ pub async fn get_all( let mut json_array_writer = JsonArrayWriter::new(); for (db_partition, db_row) in table_data.get_all_rows(skip, limit) { - update_statistics.update(db_partition, Some(db_row), now); + update_statistics.update(db_table_wrapper, db_partition, Some(db_row), now); json_array_writer.write(db_row.as_ref()); } diff --git a/src/db_operations/read/rows/get_all_by_partition_key.rs b/src/db_operations/read/rows/get_all_by_partition_key.rs index 7566ea9..5135184 100644 --- a/src/db_operations/read/rows/get_all_by_partition_key.rs +++ b/src/db_operations/read/rows/get_all_by_partition_key.rs @@ -36,7 +36,7 @@ pub async fn get_all_by_partition_key( limit, skip, |db_row| { - update_statistics.update(db_partition, Some(db_row), now); + update_statistics.update(db_table, db_partition, Some(db_row), now); }, ); diff --git a/src/db_operations/read/rows/get_all_by_row_key.rs b/src/db_operations/read/rows/get_all_by_row_key.rs index ecd65b6..2e91a48 100644 --- a/src/db_operations/read/rows/get_all_by_row_key.rs +++ b/src/db_operations/read/rows/get_all_by_row_key.rs @@ -26,7 +26,7 @@ pub async fn get_all_by_row_key( let mut json_array_writer = JsonArrayWriter::new(); for (db_partition, db_row) in table_data.get_by_row_key(row_key, skip, limit) { - update_statistics.update(db_partition, Some(db_row), now); + update_statistics.update(db_table, db_partition, Some(db_row), now); json_array_writer.write(db_row.as_ref()); } diff --git a/src/db_operations/read/rows/get_single.rs b/src/db_operations/read/rows/get_single.rs index 5dd2ef6..62642f4 100644 --- a/src/db_operations/read/rows/get_single.rs +++ b/src/db_operations/read/rows/get_single.rs @@ -38,7 +38,7 @@ pub async fn get_single( let db_row = db_row.unwrap(); - update_statistics.update(db_partition, Some(db_row), now); + update_statistics.update(db_table, db_partition, Some(db_row), now); return Ok(ReadOperationResult::SingleRow(db_row.to_vec())); } diff --git a/src/db_operations/read/rows/get_single_partition_multiple_rows.rs b/src/db_operations/read/rows/get_single_partition_multiple_rows.rs index 79f0715..ca285bd 100644 --- a/src/db_operations/read/rows/get_single_partition_multiple_rows.rs +++ b/src/db_operations/read/rows/get_single_partition_multiple_rows.rs @@ -35,7 +35,7 @@ pub async fn get_single_partition_multiple_rows( let db_row = db_partition.get_row(row_key); if let Some(db_row) = db_row { - update_statistics.update(db_partition, Some(db_row), now); + update_statistics.update(db_table_wrapper, db_partition, Some(db_row), now); json_array_writer.write(db_row.as_ref()); } } diff --git a/src/db_operations/update_statistics/update_partition_expiration_time.rs b/src/db_operations/update_statistics/update_partition_expiration_time.rs index 863909a..943bfce 100644 --- a/src/db_operations/update_statistics/update_partition_expiration_time.rs +++ b/src/db_operations/update_statistics/update_partition_expiration_time.rs @@ -3,32 +3,18 @@ use std::sync::Arc; use my_no_sql_sdk::core::rust_extensions::date_time::DateTimeAsMicroseconds; use my_no_sql_server_core::DbTableWrapper; -use crate::app::AppContext; - pub fn update_partition_expiration_time( - app: &Arc, db_table: &Arc, - partition_key: &str, + partition_key: String, set_expiration_time: Option, ) { - let partition_key = partition_key.to_string(); - let db_table = db_table.clone(); - let app = app.clone(); - tokio::spawn(async move { let mut table_data = db_table.data.write().await; - if let Some(db_partition) = table_data.get_partition_mut(&partition_key) { + if let Some(db_partition) = table_data.get_partition_mut(partition_key.as_str()) { db_partition.expires = set_expiration_time; } - - let mut sync_moment = DateTimeAsMicroseconds::now(); - sync_moment.add_minutes(5); - - app.persist_markers - .persist_partition(&table_data, &partition_key, sync_moment) - .await; }); } diff --git a/src/db_operations/update_statistics/update_rows_expiration_time.rs b/src/db_operations/update_statistics/update_rows_expiration_time.rs index 05fbe9c..c73ed78 100644 --- a/src/db_operations/update_statistics/update_rows_expiration_time.rs +++ b/src/db_operations/update_statistics/update_rows_expiration_time.rs @@ -25,6 +25,8 @@ pub fn update_rows_expiration_time<'s, TRowKeys: Iterator>( let mut updated_db_rows = Vec::new(); + let mut db_partition_key = None; + if let Some(db_partition) = table_data.get_partition_mut(&partition_key) { for row_key in row_keys { let db_row = db_partition @@ -33,14 +35,25 @@ pub fn update_rows_expiration_time<'s, TRowKeys: Iterator>( if let Some(db_row) = db_row { updated_db_rows.push(db_row); + + if db_partition_key.is_none() { + db_partition_key = Some(db_partition.partition_key.clone()); + } } } + } + if let Some(db_partition_key) = db_partition_key { let mut sync_moment = DateTimeAsMicroseconds::now(); - sync_moment.add_minutes(5); + sync_moment.add_seconds(10); app.persist_markers - .persist_partition(&table_data, &partition_key, sync_moment) + .persist_rows( + &table_data.name, + &db_partition_key, + sync_moment, + updated_db_rows.iter(), + ) .await; } }); diff --git a/src/db_operations/update_statistics/update_statistics.rs b/src/db_operations/update_statistics/update_statistics.rs index 52af3a0..800c67b 100644 --- a/src/db_operations/update_statistics/update_statistics.rs +++ b/src/db_operations/update_statistics/update_statistics.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use my_no_sql_sdk::core::db::{DbPartition, DbRow}; use my_no_sql_sdk::core::rust_extensions::date_time::DateTimeAsMicroseconds; +use my_no_sql_server_core::DbTableWrapper; #[derive(Clone, Debug)] pub struct UpdateStatistics { @@ -14,6 +15,7 @@ pub struct UpdateStatistics { impl UpdateStatistics { pub fn update( &self, + db_table: &Arc, db_partition: &DbPartition, db_row: Option<&Arc>, now: DateTimeAsMicroseconds, @@ -22,12 +24,13 @@ impl UpdateStatistics { db_partition.update_last_read_moment(now); } - /* - todo!("Not Implemented yet"); if let Some(update_partition_expiration_time) = self.update_partition_expiration_time { - db_partition.update_expires(update_partition_expiration_time); + crate::db_operations::update_statistics::update_partition_expiration_time( + db_table, + db_partition.partition_key.to_string(), + update_partition_expiration_time, + ); } - */ if let Some(db_row) = db_row { if self.update_rows_last_read_access_time { diff --git a/src/db_operations/write/bulk_delete.rs b/src/db_operations/write/bulk_delete.rs index 52a9e8a..f8bb891 100644 --- a/src/db_operations/write/bulk_delete.rs +++ b/src/db_operations/write/bulk_delete.rs @@ -27,14 +27,23 @@ pub async fn bulk_delete( table_data.bulk_remove_rows(&partition_key, row_keys.into_iter(), true, Some(now)); if let Some((partition_key, removed_rows, partition_is_empty)) = removed_rows_result { - app.persist_markers - .persist_partition(&table_data, &partition_key, persist_moment) - .await; - if partition_is_empty { sync_data.new_deleted_partition(&partition_key); + + app.persist_markers + .persist_partition(&table_data.name, &partition_key, persist_moment) + .await; } else { sync_data.add_deleted_rows(&partition_key, &removed_rows); + + app.persist_markers + .delete_db_rows( + &table_data.name, + &partition_key, + persist_moment, + removed_rows.iter(), + ) + .await; } } } diff --git a/src/db_operations/write/bulk_insert_or_update.rs b/src/db_operations/write/bulk_insert_or_update.rs index 1c121e9..02559d1 100644 --- a/src/db_operations/write/bulk_insert_or_update.rs +++ b/src/db_operations/write/bulk_insert_or_update.rs @@ -33,7 +33,12 @@ pub async fn execute( has_insert_or_replace = true; app.persist_markers - .persist_partition(&table_data, &partition_key, persist_moment) + .persist_rows( + &table_data.name, + &partition_key, + persist_moment, + db_rows.iter(), + ) .await; update_rows_state diff --git a/src/db_operations/write/clean_partition_and_bulk_insert.rs b/src/db_operations/write/clean_partition_and_bulk_insert.rs index a61fbea..bc143a0 100644 --- a/src/db_operations/write/clean_partition_and_bulk_insert.rs +++ b/src/db_operations/write/clean_partition_and_bulk_insert.rs @@ -35,7 +35,7 @@ pub async fn execute( for partition_key in partition_keys { app.persist_markers - .persist_partition(&table_data, &partition_key, persist_moment) + .persist_partition(&table_data.name, &partition_key, persist_moment) .await; let state = InitPartitionsSyncEventData::new_as_update_partition( diff --git a/src/db_operations/write/clean_table.rs b/src/db_operations/write/clean_table.rs index 69c1167..9797bd2 100644 --- a/src/db_operations/write/clean_table.rs +++ b/src/db_operations/write/clean_table.rs @@ -26,7 +26,7 @@ pub async fn execute( crate::operations::sync::dispatch(app, SyncEvent::InitTable(sync_data)); app.persist_markers - .persist_table(&table_data, persist_moment) + .persist_table_content(&table_data.name, persist_moment) .await; } diff --git a/src/db_operations/write/clean_table_and_bulk_insert.rs b/src/db_operations/write/clean_table_and_bulk_insert.rs index b0c86ce..6f71af6 100644 --- a/src/db_operations/write/clean_table_and_bulk_insert.rs +++ b/src/db_operations/write/clean_table_and_bulk_insert.rs @@ -28,7 +28,7 @@ pub async fn execute( } app.persist_markers - .persist_table(&table_data, persist_moment) + .persist_table_content(&table_data.name, persist_moment) .await; if let Some(event_src) = event_src { diff --git a/src/db_operations/write/delete_partitions.rs b/src/db_operations/write/delete_partitions.rs index d77e760..5f11080 100644 --- a/src/db_operations/write/delete_partitions.rs +++ b/src/db_operations/write/delete_partitions.rs @@ -27,9 +27,13 @@ pub async fn delete_partitions( let remove_partition_result = table_write_access.remove_partition(&partition_key, Some(now)); - if remove_partition_result.is_some() { + if let Some(removed_partition) = remove_partition_result { app.persist_markers - .persist_partition(&table_write_access, &partition_key, persist_moment) + .persist_partition( + &db_table.name, + &removed_partition.partition_key, + persist_moment, + ) .await; sync_data.add(partition_key.into_partition_key(), None); diff --git a/src/db_operations/write/delete_row.rs b/src/db_operations/write/delete_row.rs index 41903c5..7360a56 100644 --- a/src/db_operations/write/delete_row.rs +++ b/src/db_operations/write/delete_row.rs @@ -35,7 +35,12 @@ pub async fn execute( let mut sync_data = DeleteRowsEventSyncData::new(&table_data, event_src); app.persist_markers - .persist_partition(&table_data, &partition_key, persist_moment) + .persist_rows( + &table_data.name, + &partition_key, + persist_moment, + [&removed_row].into_iter(), + ) .await; if partition_is_empty { diff --git a/src/db_operations/write/insert.rs b/src/db_operations/write/insert.rs index 46b6480..5269b60 100644 --- a/src/db_operations/write/insert.rs +++ b/src/db_operations/write/insert.rs @@ -52,7 +52,12 @@ pub async fn execute( let partition_key = partition_key.unwrap(); app.persist_markers - .persist_partition(&table_data, &partition_key, persist_moment) + .persist_rows( + &table_data.name, + &partition_key, + persist_moment, + [&db_row].into_iter(), + ) .await; let mut update_rows_state = UpdateRowsSyncData::new(&table_data, event_src); diff --git a/src/db_operations/write/insert_or_replace.rs b/src/db_operations/write/insert_or_replace.rs index 2182151..cd0d7f9 100644 --- a/src/db_operations/write/insert_or_replace.rs +++ b/src/db_operations/write/insert_or_replace.rs @@ -27,7 +27,12 @@ pub async fn execute( let (partition_key, _) = table_data.insert_or_replace_row(&db_row, Some(now)); app.persist_markers - .persist_partition(&table_data, &partition_key, persist_moment) + .persist_rows( + &table_data.name, + &partition_key, + persist_moment, + [&db_row].into_iter(), + ) .await; let mut update_rows_state = UpdateRowsSyncData::new(&table_data, event_src); diff --git a/src/db_operations/write/replace.rs b/src/db_operations/write/replace.rs index 277a617..22fc614 100644 --- a/src/db_operations/write/replace.rs +++ b/src/db_operations/write/replace.rs @@ -90,7 +90,12 @@ pub async fn execute( table_data.insert_row(&db_row, Some(now.date_time)); app.persist_markers - .persist_partition(&table_data, &db_row, persist_moment) + .persist_rows( + &table_data.name, + &partition_key, + persist_moment, + [&db_row].into_iter(), + ) .await; let mut update_rows_state = UpdateRowsSyncData::new(&table_data, event_src); diff --git a/src/db_operations/write/table.rs b/src/db_operations/write/table.rs index d775df8..d287a4d 100644 --- a/src/db_operations/write/table.rs +++ b/src/db_operations/write/table.rs @@ -48,11 +48,11 @@ pub async fn create( let table_data = db_table.data.write().await; app.persist_markers - .persist_table(&table_data, persist_moment) + .persist_table_attributes(&table_data.name, persist_moment) .await; app.persist_markers - .persist_table(&table_data, persist_moment) + .persist_table_attributes(&table_data.name, persist_moment) .await; let state = InitTableEventSyncData::new(&table_data, event_src); @@ -99,10 +99,8 @@ async fn get_or_create( crate::operations::sync::dispatch(app, SyncEvent::InitTable(state)); app.persist_markers - .persist_table(&table_data, persist_moment) + .persist_table_attributes(&table_data.name, persist_moment) .await; - - app.persist_markers.persist_table_attrs(&table_data).await; } return Ok(db_table); @@ -195,13 +193,15 @@ pub async fn set_table_attributes( app, SyncEvent::UpdateTableAttributes(UpdateTableAttributesSyncData { table_data: SyncTableData { - table_name: db_table.name.to_string(), + table_name: db_table.name.clone(), }, event_src, }), ); - app.persist_markers.persist_table_attrs(&write_access).await; + app.persist_markers + .persist_table_attributes(&write_access.name, DateTimeAsMicroseconds::now()) + .await; Ok(()) } @@ -216,7 +216,7 @@ pub async fn delete( let result = app.db.delete_table(table_name.as_str()).await; if result.is_none() { - return Err(DbOperationError::TableNotFound(table_name.to_string())); + return Err(DbOperationError::TableNotFound(table_name)); } let db_table = result.unwrap(); @@ -225,7 +225,7 @@ pub async fn delete( let table_data = db_table.data.read().await; app.persist_markers - .persist_table(&table_data, persist_moment) + .persist_table_attributes(&table_data.name, persist_moment) .await; crate::operations::sync::dispatch( @@ -235,21 +235,21 @@ pub async fn delete( } let app = app.clone(); - let table_name = table_name.to_string(); - tokio::spawn( - async move { crate::persist_operations::sync::delete_table(app, table_name).await }, - ); + let table_name = db_table.name.clone(); + tokio::spawn(async move { + crate::operations::persist::scripts::delete_table(&app, &table_name).await + }); Ok(()) } -pub async fn init(app: &AppContext, db_table: DbTable) { - app.blob_content_cache.init(&db_table).await; - +pub async fn init(app: &AppContext, db_table: DbTable) -> Arc { let db_table = DbTableWrapper::new(db_table); let mut tables_write_access = app.db.tables.write().await; - tables_write_access.insert(db_table.name.to_string(), db_table); + tables_write_access.insert(db_table.name.to_string(), db_table.clone()); + + db_table } enum CreateTableResult { @@ -278,7 +278,7 @@ async fn get_or_create_table( max_rows_per_partition_amount, }; - let db_table = DbTable::new(table_name.to_string(), table_attributes); + let db_table = DbTable::new(table_name.into(), table_attributes); let db_table_wrapper = DbTableWrapper::new(db_table); diff --git a/src/db_sync/states/sync_table_data.rs b/src/db_sync/states/sync_table_data.rs index ae6bdb3..3942dba 100644 --- a/src/db_sync/states/sync_table_data.rs +++ b/src/db_sync/states/sync_table_data.rs @@ -1,13 +1,13 @@ -use my_no_sql_sdk::core::db::DbTable; +use my_no_sql_sdk::core::db::{DbTable, DbTableName}; pub struct SyncTableData { - pub table_name: String, + pub table_name: DbTableName, } impl SyncTableData { pub fn new(table_data: &DbTable) -> Self { Self { - table_name: table_data.name.to_string(), + table_name: table_data.name.clone(), } } } diff --git a/src/db_sync/sync_event.rs b/src/db_sync/sync_event.rs index 2aa2073..7e1d885 100644 --- a/src/db_sync/sync_event.rs +++ b/src/db_sync/sync_event.rs @@ -23,13 +23,13 @@ pub enum SyncEvent { impl SyncEvent { pub fn get_table_name(&self) -> &str { match self { - SyncEvent::UpdateTableAttributes(data) => data.table_data.table_name.as_ref(), - SyncEvent::InitTable(data) => data.table_data.table_name.as_ref(), - SyncEvent::InitPartitions(data) => data.table_data.table_name.as_ref(), - SyncEvent::UpdateRows(data) => data.table_data.table_name.as_ref(), - SyncEvent::DeleteRows(data) => data.table_data.table_name.as_ref(), - SyncEvent::DeleteTable(data) => data.table_data.table_name.as_ref(), - SyncEvent::TableFirstInit(data) => data.db_table.name.as_ref(), + SyncEvent::UpdateTableAttributes(data) => data.table_data.table_name.as_str(), + SyncEvent::InitTable(data) => data.table_data.table_name.as_str(), + SyncEvent::InitPartitions(data) => data.table_data.table_name.as_str(), + SyncEvent::UpdateRows(data) => data.table_data.table_name.as_str(), + SyncEvent::DeleteRows(data) => data.table_data.table_name.as_str(), + SyncEvent::DeleteTable(data) => data.table_data.table_name.as_str(), + SyncEvent::TableFirstInit(data) => data.db_table.name.as_str(), } } } diff --git a/src/http/controllers/data_reader_controller/get_changes_action.rs b/src/http/controllers/data_reader_controller/get_changes_action.rs index 8108a0b..760cbae 100644 --- a/src/http/controllers/data_reader_controller/get_changes_action.rs +++ b/src/http/controllers/data_reader_controller/get_changes_action.rs @@ -96,9 +96,8 @@ async fn update_expiration_time( .to_set_expiration_time() { crate::db_operations::update_partition_expiration_time( - app, &db_table, - &item.partition_key, + item.partition_key.to_string(), set_expiration_time, ) } diff --git a/src/http/controllers/status_controller/models.rs b/src/http/controllers/status_controller/models.rs index 8208a1a..5bd583c 100644 --- a/src/http/controllers/status_controller/models.rs +++ b/src/http/controllers/status_controller/models.rs @@ -80,7 +80,7 @@ impl StatusModel { }; let table_model = TableModel { - name: table.name.clone(), + name: table.name.to_string(), avg_entity_size: metrics.avg_entity_size, persist: attr.persist, max_partitions_amount: attr.max_partitions_amount, diff --git a/src/http/controllers/status_controller/non_initialized/model.rs b/src/http/controllers/status_controller/non_initialized/model.rs index 87a689a..b2acc93 100644 --- a/src/http/controllers/status_controller/non_initialized/model.rs +++ b/src/http/controllers/status_controller/non_initialized/model.rs @@ -9,10 +9,9 @@ pub struct NonInitializedModel { tables_total: usize, #[serde(rename = "tablesLoaded")] tables_loaded: usize, - #[serde(rename = "filesTotal")] - files_total: usize, - #[serde(rename = "filesLoaded")] - files_loaded: usize, + #[serde(rename = "currentTable")] + current_table: Option, + error: Option, #[serde(rename = "initializingSeconds")] loading_time: i64, } @@ -21,13 +20,14 @@ impl NonInitializedModel { pub async fn new(app: &AppContext) -> Self { let now = DateTimeAsMicroseconds::now(); - let snapshot = app.init_state.get_snapshot().await; + let state = app.init_state.clone().await; Self { - tables_total: snapshot.tables_total, - files_total: snapshot.files_total, - files_loaded: snapshot.files_loaded, - tables_loaded: snapshot.tables_loaded, + tables_total: state.total_tables, + tables_loaded: state.loaded, + current_table: state.current_table, + error: state.error, + loading_time: now.seconds_before(app.created), } } diff --git a/src/main.rs b/src/main.rs index b0b0330..517ce8c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,21 +13,19 @@ mod zip; mod app; mod grpc; +mod persist_markers; mod sqlite_repo; -mod persist; - mod db_operations; mod db_sync; mod db_transactions; mod http; -mod persist_operations; +mod scripts; mod tcp; mod background; mod data_readers; mod operations; -mod persist_io; mod settings_reader; //TODO - Add Amount of Subscribers to table on UI; @@ -45,15 +43,11 @@ async fn main() { let settings = Arc::new(settings); - let persist_io = settings.get_persist_io().await; - - let app = AppContext::new(settings, persist_io); + let app = AppContext::new(settings).await; let app = Arc::new(app); - tokio::spawn(crate::persist_operations::data_initializer::load_tables( - app.clone(), - )); + tokio::spawn(crate::operations::init::load_tables(app.clone())); let http_connections_counter = crate::http::start_up::setup_server(&app); diff --git a/src/operations/build_db_snapshot_as_zip.rs b/src/operations/build_db_snapshot_as_zip.rs index 1e285e9..b278f52 100644 --- a/src/operations/build_db_snapshot_as_zip.rs +++ b/src/operations/build_db_snapshot_as_zip.rs @@ -9,7 +9,7 @@ pub async fn build_db_snapshot_as_zip_archive(app: &AppContext) -> Vec { let table_snapshot = db_table.get_table_snapshot().await; zip_builder - .add_table(&db_table.name, &table_snapshot) + .add_table(&db_table.name.as_str(), &table_snapshot) .unwrap(); } diff --git a/src/operations/init/entities_from_sqlite_reader.rs b/src/operations/init/entities_from_sqlite_reader.rs new file mode 100644 index 0000000..9672026 --- /dev/null +++ b/src/operations/init/entities_from_sqlite_reader.rs @@ -0,0 +1,55 @@ +use std::{collections::HashMap, sync::Arc}; + +use my_no_sql_sdk::core::{db::DbRow, db_json_entity::DbJsonEntity}; +use my_no_sql_server_core::rust_extensions; + +use crate::sqlite_repo::MyNoSqlEntityDto; + +use super::EntitiesInitReader; + +pub struct EntitiesFromSqliteReader { + by_table: HashMap>, + skip_errors: bool, +} + +impl EntitiesFromSqliteReader { + pub fn new(entities: Vec, skip_errors: bool) -> Self { + Self { + skip_errors, + by_table: rust_extensions::grouped_data::group_to_hash_map( + entities.into_iter(), + |itm| itm.table_name.clone(), + ), + } + } +} + +#[async_trait::async_trait] +impl EntitiesInitReader for EntitiesFromSqliteReader { + async fn get_entities(&mut self, table_name: &str) -> Option>> { + let dtos = self.by_table.remove(table_name)?; + + let mut result = Vec::with_capacity(dtos.len()); + + for dto in dtos { + match DbJsonEntity::restore_into_db_row(dto.content.into_bytes()) { + Ok(db_row) => result.push(Arc::new(db_row)), + Err(err) => { + if self.skip_errors { + println!( + "Can not restore row Table:{}. PartitionKey: {} RowKey: {}. Err: {:?}", + table_name, dto.partition_key, dto.row_key, err + ); + } else { + panic!( + "Can not restore row Table:{}. PartitionKey: {} RowKey: {}. Err: {:?}", + table_name, dto.partition_key, dto.row_key, err + ); + } + } + } + } + + Some(result) + } +} diff --git a/src/operations/init/from_other_instance/load_rows.rs b/src/operations/init/from_other_instance/load_rows.rs new file mode 100644 index 0000000..ee2237d --- /dev/null +++ b/src/operations/init/from_other_instance/load_rows.rs @@ -0,0 +1,41 @@ +use std::sync::Arc; + +use flurl::FlUrl; +use my_no_sql_sdk::core::{db::DbRow, db_json_entity::DbJsonEntity}; + +use crate::operations::init::EntitiesInitReader; + +pub async fn load_rows(url: &str, table_name: &str) -> Vec> { + let mut response = FlUrl::new(url) + .append_path_segment("api") + .append_path_segment("Row") + .append_query_param("tableName", Some(table_name)) + .get() + .await + .unwrap(); + + let body = response.get_body_as_slice().await.unwrap(); + + DbJsonEntity::restore_as_vec(body).unwrap() +} + +pub struct EntitiesInitReaderForOtherInstance(String); + +impl EntitiesInitReaderForOtherInstance { + pub fn new(url: String) -> Self { + Self(url) + } +} + +#[async_trait::async_trait] +impl EntitiesInitReader for EntitiesInitReaderForOtherInstance { + async fn get_entities(&mut self, table_name: &str) -> Option>> { + let result = load_rows(self.0.as_str(), table_name).await; + + if result.len() == 0 { + return None; + } + + Some(result) + } +} diff --git a/src/operations/init/from_other_instance/load_tables.rs b/src/operations/init/from_other_instance/load_tables.rs new file mode 100644 index 0000000..1309d9c --- /dev/null +++ b/src/operations/init/from_other_instance/load_tables.rs @@ -0,0 +1,57 @@ +use flurl::FlUrl; +use my_no_sql_sdk::core::db::DbTableAttributes; +use my_no_sql_server_core::rust_extensions::date_time::DateTimeAsMicroseconds; +use serde::*; + +use crate::operations::init::TableAttributeInitContract; + +#[derive(Debug, Serialize, Deserialize)] +pub struct TableMyNoSqlServerContract { + pub name: String, + pub persist: Option, + #[serde(rename = "maxPartitionsAmount")] + pub max_partitions_amount: Option, + pub max_rows_per_partition_amount: Option, + pub created: Option, +} + +impl TableAttributeInitContract for TableMyNoSqlServerContract { + fn into( + self, + ) -> ( + my_no_sql_sdk::core::db::DbTableName, + my_no_sql_sdk::core::db::DbTableAttributes, + ) { + let created = if let Some(created) = self.created.as_ref() { + match DateTimeAsMicroseconds::from_str(created) { + Some(x) => x, + None => DateTimeAsMicroseconds::now(), + } + } else { + DateTimeAsMicroseconds::now() + }; + + let attributes = DbTableAttributes { + persist: self.persist.unwrap_or(false), + max_partitions_amount: self.max_partitions_amount.map(|x| x as usize), + max_rows_per_partition_amount: self.max_rows_per_partition_amount.map(|x| x as usize), + created, + }; + + (self.name.into(), attributes) + } +} + +pub async fn load_tables(url: &str) -> Vec { + let mut response = FlUrl::new(url) + .append_path_segment("api") + .append_path_segment("Tables") + .append_path_segment("List") + .get() + .await + .unwrap(); + + let body = response.get_body_as_slice().await.unwrap(); + + serde_json::from_slice(body).unwrap() +} diff --git a/src/persist_operations/data_initializer/from_other_instance/mod.rs b/src/operations/init/from_other_instance/mod.rs similarity index 52% rename from src/persist_operations/data_initializer/from_other_instance/mod.rs rename to src/operations/init/from_other_instance/mod.rs index 00e7f2d..f4fd613 100644 --- a/src/persist_operations/data_initializer/from_other_instance/mod.rs +++ b/src/operations/init/from_other_instance/mod.rs @@ -2,5 +2,3 @@ mod load_tables; pub use load_tables::*; mod load_rows; pub use load_rows::*; -mod init_from_another_instance; -pub use init_from_another_instance::*; diff --git a/src/operations/init/init_state.rs b/src/operations/init/init_state.rs new file mode 100644 index 0000000..dc1fda0 --- /dev/null +++ b/src/operations/init/init_state.rs @@ -0,0 +1,31 @@ +use tokio::sync::Mutex; + +#[derive(Clone)] +pub struct InitStateInner { + pub total_tables: usize, + pub loaded: usize, + pub current_table: Option, + pub error: Option, +} + +pub struct InitState { + inner: Mutex, +} + +impl InitState { + pub fn new() -> Self { + Self { + inner: Mutex::new(InitStateInner { + total_tables: 0, + loaded: 0, + current_table: None, + error: None, + }), + } + } + + pub async fn clone(&self) -> InitStateInner { + let inner = self.inner.lock().await; + inner.clone() + } +} diff --git a/src/operations/init/load_tables.rs b/src/operations/init/load_tables.rs new file mode 100644 index 0000000..393814d --- /dev/null +++ b/src/operations/init/load_tables.rs @@ -0,0 +1,44 @@ +use std::sync::Arc; + +use my_no_sql_server_core::rust_extensions::StopWatch; + +use crate::app::AppContext; + +use super::entities_from_sqlite_reader::EntitiesFromSqliteReader; + +pub async fn load_tables(app: Arc) { + let mut sw = StopWatch::new(); + sw.start(); + + if let Some(server_url) = app.settings.get_init_from_other_server_url() { + let tables = super::from_other_instance::load_tables(server_url).await; + let tables_amount = tables.len(); + + let entities_reader = super::from_other_instance::EntitiesInitReaderForOtherInstance::new( + server_url.to_string(), + ); + + super::scripts::init_tables(&app, tables, entities_reader, true).await; + + println!( + "Tables loaded: {} in {:?} seconds", + tables_amount, + sw.duration() + ); + } else { + let tables = app.repo.get_tables().await; + let tables_amount = tables.len(); + let entities = app.repo.get_all_entities().await; + let entities_reader = + EntitiesFromSqliteReader::new(entities, app.settings.skip_broken_partitions); + super::scripts::init_tables(&app, tables, entities_reader, false).await; + sw.pause(); + println!( + "Tables loaded: {} in {:?} seconds", + tables_amount, + sw.duration() + ); + } + + app.states.set_initialized(); +} diff --git a/src/operations/init/mod.rs b/src/operations/init/mod.rs new file mode 100644 index 0000000..0d5d5fa --- /dev/null +++ b/src/operations/init/mod.rs @@ -0,0 +1,20 @@ +mod load_tables; +use std::sync::Arc; + +pub use load_tables::*; +mod entities_from_sqlite_reader; +mod from_other_instance; +mod scripts; +use my_no_sql_sdk::core::db::{DbRow, DbTableAttributes, DbTableName}; + +mod init_state; +pub use init_state::*; + +pub trait TableAttributeInitContract { + fn into(self) -> (DbTableName, DbTableAttributes); +} + +#[async_trait::async_trait] +pub trait EntitiesInitReader { + async fn get_entities(&mut self, table_name: &str) -> Option>>; +} diff --git a/src/operations/init/scripts.rs b/src/operations/init/scripts.rs new file mode 100644 index 0000000..d008e1f --- /dev/null +++ b/src/operations/init/scripts.rs @@ -0,0 +1,53 @@ +use my_no_sql_sdk::core::db::{DbPartition, DbTable}; +use my_no_sql_server_core::rust_extensions; + +use crate::app::AppContext; + +use super::{EntitiesInitReader, TableAttributeInitContract}; + +pub async fn init_tables( + app: &AppContext, + tables: Vec, + mut entities_reader: impl EntitiesInitReader, + save_to_db: bool, +) { + for table_init_contract in tables { + let (table_name, attr) = table_init_contract.into(); + let mut db_table = DbTable::new(table_name, attr); + + let db_rows = entities_reader.get_entities(db_table.name.as_str()).await; + + if let Some(db_rows) = db_rows { + let by_partition = + rust_extensions::grouped_data::group_to_btree_map(db_rows.into_iter(), |itm| { + itm.get_partition_key().to_string() + }); + + for (partition_key, entities) in by_partition { + let mut db_partition = DbPartition::new(partition_key); + for db_row in entities { + db_partition.insert_row(db_row); + } + + db_table.restore_partition(db_partition); + } + } + + let db_table = crate::db_operations::write::table::init(app, db_table).await; + + if save_to_db { + let table_snapshot = db_table.get_table_snapshot().await; + + app.repo + .save_table_metadata(&db_table.name, &table_snapshot.attr) + .await; + + crate::operations::persist::scripts::sync_table_snapshot( + app, + &db_table.name, + table_snapshot, + ) + .await; + } + } +} diff --git a/src/operations/mod.rs b/src/operations/mod.rs index a9c9433..15cac11 100644 --- a/src/operations/mod.rs +++ b/src/operations/mod.rs @@ -1,13 +1,14 @@ pub mod data_readers; mod get_metrics; -mod persist; +pub mod persist; pub mod shutdown; pub mod sync; pub use get_metrics::*; -pub use persist::persist; +pub use persist::*; mod build_db_snapshot_as_zip; pub use build_db_snapshot_as_zip::*; pub mod backup; mod parse_db_json_entity; pub use parse_db_json_entity::*; +pub mod init; diff --git a/src/operations/persist.rs b/src/operations/persist.rs deleted file mode 100644 index 994bfea..0000000 --- a/src/operations/persist.rs +++ /dev/null @@ -1,84 +0,0 @@ -use std::sync::Arc; - -use my_logger::LogEventCtx; -use my_no_sql_sdk::core::rust_extensions::{date_time::DateTimeAsMicroseconds, StopWatch}; -use my_no_sql_server_core::DbTableWrapper; - -use crate::{app::AppContext, persist::partition_persist_marker::PersistResult}; - -pub async fn persist(app: &Arc) { - let is_shutting_down = app.states.is_shutting_down(); - - loop { - let tables = app.db.get_tables().await; - - let mut has_something_to_persist = false; - - for db_table in tables { - if let Some(persist_result) = app - .persist_markers - .get_job_to_persist( - db_table.name.as_str(), - DateTimeAsMicroseconds::now(), - is_shutting_down, - ) - .await - { - has_something_to_persist = true; - let mut sw = StopWatch::new(); - sw.start(); - let result = tokio::spawn(persist_to_blob( - app.clone(), - db_table.clone(), - persist_result, - )) - .await; - - sw.pause(); - - if result.is_ok() { - app.persist_markers - .set_persisted(db_table.name.as_str(), sw.duration()) - .await; - } - - if let Err(err) = result { - my_logger::LOGGER.write_fatal_error( - "PersistTimer".to_string(), - format!("Can not persist messages {:?}", err), - LogEventCtx::new().add("table_name", db_table.name.as_str()), - ) - } - } - } - - if !has_something_to_persist { - break; - } - } -} - -async fn persist_to_blob( - app: Arc, - db_table: Arc, - persist_result: PersistResult, -) { - match persist_result { - PersistResult::PersistAttrs => { - let attrs = db_table.get_attributes().await; - crate::persist_operations::sync::save_table_attributes( - app.as_ref(), - db_table.name.as_str(), - &attrs, - ) - .await; - } - PersistResult::PersistTable => { - crate::persist_operations::sync::save_table(app.as_ref(), db_table.as_ref()).await; - } - PersistResult::PersistPartition(partition_key) => { - crate::persist_operations::sync::save_partition(app.as_ref(), &db_table, partition_key) - .await; - } - } -} diff --git a/src/operations/persist/mod.rs b/src/operations/persist/mod.rs new file mode 100644 index 0000000..3d6ddab --- /dev/null +++ b/src/operations/persist/mod.rs @@ -0,0 +1,12 @@ +mod persist; +pub use persist::*; +mod save_table_attributes; +use save_table_attributes::*; +mod save_table; +pub mod scripts; +use save_table::*; +mod save_partition; +use save_partition::*; +mod save_rows; +use save_rows::*; +const SAVE_ENTITIES_BATCH_SIZE: usize = 200; diff --git a/src/operations/persist/persist.rs b/src/operations/persist/persist.rs new file mode 100644 index 0000000..dbe14ed --- /dev/null +++ b/src/operations/persist/persist.rs @@ -0,0 +1,53 @@ +use std::sync::Arc; + +use my_no_sql_sdk::core::rust_extensions::date_time::DateTimeAsMicroseconds; + +use crate::{app::AppContext, persist_markers::PersistTask}; + +pub async fn persist(app: &Arc) { + loop { + let start_time = DateTimeAsMicroseconds::now(); + + let now = if app.states.is_shutting_down() { + None + } else { + Some(start_time) + }; + + let persist_task = + if let Some(persist_task) = app.persist_markers.get_persist_task(now).await { + persist_task + } else { + return; + }; + + let db_table_name = match persist_task { + PersistTask::SaveTableAttributes(db_table_name) => { + super::save_table_attributes(app, &db_table_name).await; + db_table_name + } + PersistTask::SyncTable(db_table_name) => { + super::save_table(app, &db_table_name).await; + db_table_name + } + PersistTask::SyncPartition { + table_name, + partition_key, + } => { + super::save_partition(app, &table_name, partition_key).await; + table_name + } + PersistTask::SyncRows { table_name, jobs } => { + super::save_rows(app, &table_name, jobs).await; + table_name + } + }; + + let now = DateTimeAsMicroseconds::now(); + let duration = now.duration_since(start_time).as_positive_or_zero(); + + app.persist_markers + .set_last_persist_time(&db_table_name, now, duration) + .await; + } +} diff --git a/src/operations/persist/save_partition.rs b/src/operations/persist/save_partition.rs new file mode 100644 index 0000000..a61a1d5 --- /dev/null +++ b/src/operations/persist/save_partition.rs @@ -0,0 +1,31 @@ +use my_no_sql_sdk::core::db::{DbTableName, PartitionKey}; + +use crate::app::AppContext; + +pub async fn save_partition( + app: &AppContext, + db_table_name: &DbTableName, + partition_key: PartitionKey, +) { + let db_table = match app.db.get_table(db_table_name.as_str()).await { + Some(db_table) => db_table, + None => { + super::scripts::delete_table(app, db_table_name).await; + return; + } + }; + + match db_table + .get_partition_snapshot(partition_key.as_str()) + .await + { + Some(snapshot) => { + super::scripts::sync_partition_snapshot(app, db_table_name, &partition_key, snapshot) + .await; + } + None => { + super::scripts::delete_partition(app, db_table_name, &partition_key).await; + return; + } + }; +} diff --git a/src/operations/persist/save_rows.rs b/src/operations/persist/save_rows.rs new file mode 100644 index 0000000..50fa2a9 --- /dev/null +++ b/src/operations/persist/save_rows.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; + +use my_no_sql_sdk::core::db::{DbRow, DbTableName, PartitionKey}; +use my_no_sql_server_core::DbTableWrapper; + +use crate::{app::AppContext, persist_markers::SyncRowJobDescription}; + +pub async fn save_rows( + app: &AppContext, + db_table_name: &DbTableName, + jobs: Vec, +) { + let db_table = match app.db.get_table(db_table_name.as_str()).await { + Some(db_table) => db_table, + None => { + super::scripts::delete_table(app, &db_table_name).await; + return; + } + }; + + let found_rows = find_and_sort_rows(&db_table, jobs).await; + + for partition_to_delete in found_rows.partitions_to_delete { + super::scripts::delete_partition(app, db_table_name, &partition_to_delete).await; + } + + if found_rows.rows_to_delete.len() > 0 { + super::scripts::delete_rows(app, db_table_name, &found_rows.rows_to_delete).await; + } + + if found_rows.rows_to_save.len() > 0 { + super::scripts::save_rows(app, db_table_name, &found_rows.rows_to_save).await; + } +} + +async fn find_and_sort_rows( + db_table: &DbTableWrapper, + jobs: Vec, +) -> FoundRowsToSync { + let mut result = FoundRowsToSync::default(); + let db_table_access = db_table.data.read().await; + for job in jobs { + match db_table_access.partitions.get(job.partition_key.as_str()) { + Some(db_partition) => { + for job_db_row in job.items { + match db_partition.get_row(job_db_row.get_row_key()) { + Some(db_row) => { + result.rows_to_save.push(db_row.clone()); + } + None => { + result.rows_to_delete.push(job_db_row); + } + } + } + } + None => { + result.partitions_to_delete.push(job.partition_key.clone()); + } + } + } + result +} + +#[derive(Default)] +pub struct FoundRowsToSync { + rows_to_save: Vec>, + rows_to_delete: Vec>, + partitions_to_delete: Vec, +} diff --git a/src/operations/persist/save_table.rs b/src/operations/persist/save_table.rs new file mode 100644 index 0000000..477bc3b --- /dev/null +++ b/src/operations/persist/save_table.rs @@ -0,0 +1,15 @@ +use my_no_sql_sdk::core::db::DbTableName; + +use crate::app::AppContext; + +pub async fn save_table(app: &AppContext, table_name: &DbTableName) { + match app.db.get_table(table_name.as_str()).await { + Some(db_table) => { + let table_snapshot = db_table.get_table_snapshot().await; + super::scripts::sync_table_snapshot(app, table_name, table_snapshot).await; + } + None => { + super::scripts::delete_table(app, &table_name).await; + } + } +} diff --git a/src/operations/persist/save_table_attributes.rs b/src/operations/persist/save_table_attributes.rs new file mode 100644 index 0000000..a937b6d --- /dev/null +++ b/src/operations/persist/save_table_attributes.rs @@ -0,0 +1,15 @@ +use my_no_sql_sdk::core::db::DbTableName; + +use crate::app::AppContext; + +pub async fn save_table_attributes(app: &AppContext, table_name: &DbTableName) { + match app.db.get_table(table_name.as_str()).await { + Some(db_table) => { + let attr = db_table.get_attributes().await; + app.repo.save_table_metadata(&table_name, &attr).await; + } + None => { + super::scripts::delete_table(app, &table_name).await; + } + } +} diff --git a/src/operations/persist/scripts.rs b/src/operations/persist/scripts.rs new file mode 100644 index 0000000..3de2a9e --- /dev/null +++ b/src/operations/persist/scripts.rs @@ -0,0 +1,88 @@ +use std::sync::Arc; + +use my_no_sql_sdk::core::db::{DbRow, DbTableName, PartitionKey}; +use my_no_sql_server_core::db_snapshots::{DbPartitionSnapshot, DbTableSnapshot}; + +use crate::{app::AppContext, sqlite_repo::MyNoSqlEntityDto}; + +pub async fn delete_partition( + app: &AppContext, + table_name: &DbTableName, + partition_key: &PartitionKey, +) { + app.repo + .clean_partition_content(table_name, partition_key) + .await; +} + +pub async fn delete_table(app: &AppContext, table_name: &DbTableName) { + app.repo.clean_table_content(table_name).await; + app.repo.delete_table_metadata(table_name).await; +} + +pub async fn sync_table_snapshot( + app: &AppContext, + table_name: &DbTableName, + table_snapshot: DbTableSnapshot, +) { + app.repo.clean_table_content(table_name).await; + + let mut rows_to_save = Vec::new(); + for partition_snapshot in table_snapshot.by_partition { + for db_row in partition_snapshot.db_rows_snapshot.db_rows.iter() { + rows_to_save.push(MyNoSqlEntityDto::from_db_row(table_name.as_str(), db_row)); + + if rows_to_save.len() >= super::SAVE_ENTITIES_BATCH_SIZE { + app.repo.save_entities(&rows_to_save).await; + rows_to_save.clear(); + } + } + } + + if rows_to_save.len() > 0 { + app.repo.save_entities(&rows_to_save).await; + } +} + +pub async fn sync_partition_snapshot( + app: &AppContext, + table_name: &DbTableName, + partition_key: &PartitionKey, + partition_snapshot: DbPartitionSnapshot, +) { + app.repo + .clean_partition_content(table_name, partition_key) + .await; + + save_rows( + app, + table_name, + &partition_snapshot.db_rows_snapshot.db_rows, + ) + .await; +} + +pub async fn delete_rows(app: &AppContext, table_name: &DbTableName, db_rows: &[Arc]) { + for db_row in db_rows { + app.repo + .delete_entity(table_name, db_row.get_partition_key(), db_row.get_row_key()) + .await; + } +} + +pub async fn save_rows(app: &AppContext, table_name: &DbTableName, db_rows: &[Arc]) { + let mut rows_to_save = Vec::new(); + + for db_row in db_rows.iter() { + rows_to_save.push(MyNoSqlEntityDto::from_db_row(table_name.as_str(), db_row)); + + if rows_to_save.len() >= super::SAVE_ENTITIES_BATCH_SIZE { + app.repo.save_entities(&rows_to_save).await; + rows_to_save.clear(); + } + } + + if rows_to_save.len() > 0 { + app.repo.save_entities(&rows_to_save).await; + } +} diff --git a/src/persist/mod.rs b/src/persist/mod.rs deleted file mode 100644 index 952b52b..0000000 --- a/src/persist/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod partition_persist_marker; -mod persist_markers_by_table; -pub use persist_markers_by_table::PersistMarkersByTable; -mod persist_markers; -pub use persist_markers::*; diff --git a/src/persist/persist_markers.rs b/src/persist/persist_markers.rs deleted file mode 100644 index aae9369..0000000 --- a/src/persist/persist_markers.rs +++ /dev/null @@ -1,32 +0,0 @@ -use std::time::Duration; - -use my_no_sql_sdk::core::rust_extensions::date_time::DateTimeAsMicroseconds; - -use super::partition_persist_marker::PartitionPersistMarker; -const DURATION_MONITORING_DATA_SIZE: usize = 120; - -pub struct TablePersistData { - pub data_to_persist: PartitionPersistMarker, - pub persist_duration: Vec, - pub last_persist_time: Option, -} - -impl TablePersistData { - pub fn new() -> Self { - Self { - data_to_persist: PartitionPersistMarker::new(), - persist_duration: Vec::with_capacity(DURATION_MONITORING_DATA_SIZE), - last_persist_time: None, - } - } - - pub fn add_persist_duration(&mut self, dur: Duration) { - while self.persist_duration.len() == DURATION_MONITORING_DATA_SIZE { - self.persist_duration.remove(0); - } - - self.persist_duration.push(dur.as_micros() as usize); - - self.last_persist_time = DateTimeAsMicroseconds::now().into(); - } -} diff --git a/src/persist/persist_markers_by_table.rs b/src/persist/persist_markers_by_table.rs deleted file mode 100644 index a136550..0000000 --- a/src/persist/persist_markers_by_table.rs +++ /dev/null @@ -1,181 +0,0 @@ -use std::time::Duration; - -use my_no_sql_sdk::core::db::{DbTable, PartitionKeyParameter}; -use my_no_sql_sdk::core::rust_extensions::{ - date_time::DateTimeAsMicroseconds, - sorted_vec::{EntityWithStrKey, SortedVecWithStrKey}, -}; -use tokio::sync::Mutex; - -use super::{partition_persist_marker::PersistResult, TablePersistData}; - -pub struct PersistMetrics { - pub last_persist_time: Option, - pub next_persist_time: Option, - pub persist_amount: usize, - pub last_persist_duration: Vec, -} - -pub struct PersistByTableItem { - pub table_name: String, - pub data: TablePersistData, -} - -impl EntityWithStrKey for PersistByTableItem { - fn get_key(&self) -> &str { - self.table_name.as_str() - } -} - -pub struct PersistMarkersByTable { - by_table: Mutex>, -} - -impl PersistMarkersByTable { - pub fn new() -> Self { - Self { - by_table: Mutex::new(SortedVecWithStrKey::new()), - } - } - - pub async fn persist_partition( - &self, - db_table: &DbTable, - partition_key: &impl PartitionKeyParameter, - sync_moment: DateTimeAsMicroseconds, - ) { - if !db_table.attributes.persist { - return; - } - - let mut write_access = self.by_table.lock().await; - - match write_access.insert_or_update(db_table.name.as_str()) { - my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Insert( - entry, - ) => { - let mut item = PersistByTableItem { - table_name: db_table.name.clone(), - data: TablePersistData::new(), - }; - - item.data - .data_to_persist - .mark_partition_to_persist(partition_key, sync_moment); - - entry.insert(item); - } - my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Update( - entry, - ) => { - entry - .item - .data - .data_to_persist - .mark_partition_to_persist(partition_key, sync_moment); - } - } - } - - pub async fn persist_table(&self, db_table: &DbTable, sync_moment: DateTimeAsMicroseconds) { - if !db_table.attributes.persist { - return; - } - - let mut write_access = self.by_table.lock().await; - - match write_access.insert_or_update(&db_table.name) { - my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Insert( - entry, - ) => { - let mut item = PersistByTableItem { - table_name: db_table.name.to_string(), - data: TablePersistData::new(), - }; - - item.data.data_to_persist.mark_table_to_persist(sync_moment); - - entry.insert(item); - } - my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Update( - entry, - ) => { - entry - .item - .data - .data_to_persist - .mark_table_to_persist(sync_moment); - } - } - } - - pub async fn persist_table_attrs(&self, db_table: &DbTable) { - if !db_table.attributes.persist { - return; - } - - let mut write_access = self.by_table.lock().await; - - match write_access.insert_or_update(&db_table.name) { - my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Insert( - entry, - ) => { - let mut item = PersistByTableItem { - table_name: db_table.name.to_string(), - data: TablePersistData::new(), - }; - - item.data.data_to_persist.mark_persist_attrs(); - - entry.insert(item); - } - my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Update( - entry, - ) => { - entry.item.data.data_to_persist.mark_persist_attrs(); - } - } - } - - pub async fn get_job_to_persist( - &self, - table_name: &str, - now: DateTimeAsMicroseconds, - is_shutting_down: bool, - ) -> Option { - let mut write_access = self.by_table.lock().await; - - let item = write_access.get_mut(table_name)?; - - item.data - .data_to_persist - .get_what_to_persist(now, is_shutting_down) - } - - pub async fn set_persisted(&self, table_name: &str, duration: Duration) { - let mut write_access = self.by_table.lock().await; - - if let Some(item) = write_access.get_mut(table_name) { - item.data.add_persist_duration(duration); - } - } - - pub async fn get_persist_metrics(&self, table_name: &str) -> PersistMetrics { - let read_access = self.by_table.lock().await; - - match read_access.get(table_name) { - Some(result) => PersistMetrics { - last_persist_time: result.data.last_persist_time.clone(), - next_persist_time: result.data.data_to_persist.get_next_persist_time(), - persist_amount: result.data.data_to_persist.get_persist_amount(), - last_persist_duration: result.data.persist_duration.clone(), - }, - None => PersistMetrics { - last_persist_time: None, - next_persist_time: None, - persist_amount: 0, - last_persist_duration: vec![], - }, - } - } -} diff --git a/src/persist_io/azure/attempt_handling.rs b/src/persist_io/azure/attempt_handling.rs deleted file mode 100644 index c807ecd..0000000 --- a/src/persist_io/azure/attempt_handling.rs +++ /dev/null @@ -1,26 +0,0 @@ -use std::time::Duration; - -use my_logger::LogEventCtx; - -pub async fn execute( - table_name: Option<&str>, - process_name: &str, - message: String, - attempt_no: u8, -) { - if attempt_no >= 5 { - panic!("{}", message.as_str()); - } - - let ctx = if let Some(table_name) = table_name { - LogEventCtx::new() - .add("tableName", table_name) - .add("attempt", attempt_no.to_string()) - } else { - LogEventCtx::new().add("attempt", attempt_no.to_string()) - }; - - my_logger::LOGGER.write_error(process_name.to_string(), message, ctx); - - tokio::time::sleep(Duration::from_secs(1)).await; -} diff --git a/src/persist_io/azure/create_table.rs b/src/persist_io/azure/create_table.rs deleted file mode 100644 index dff1aa2..0000000 --- a/src/persist_io/azure/create_table.rs +++ /dev/null @@ -1,19 +0,0 @@ -use my_azure_storage_sdk::{blob_container::BlobContainersApi, AzureStorageConnection}; - -pub async fn create_table(azure_connection: &AzureStorageConnection, table_name: &str) { - let mut attempt_no = 0; - while let Err(err) = azure_connection - .create_container_if_not_exists(table_name) - .await - { - super::attempt_handling::execute( - Some(table_name), - "create_table", - format!("Error creating table [{}]. Err: {:?}", table_name, err), - attempt_no, - ) - .await; - - attempt_no += 1; - } -} diff --git a/src/persist_io/azure/delete_table_file.rs b/src/persist_io/azure/delete_table_file.rs deleted file mode 100644 index b76c206..0000000 --- a/src/persist_io/azure/delete_table_file.rs +++ /dev/null @@ -1,28 +0,0 @@ -use my_azure_storage_sdk::blob::BlobApi; - -use my_azure_storage_sdk::AzureStorageConnection; - -pub async fn delete_table_file( - azure_connection: &AzureStorageConnection, - table_name: &str, - blob_name: &str, -) { - let mut attempt_no = 0; - - while let Err(err) = azure_connection - .delete_blob_if_exists(table_name, blob_name) - .await - { - super::attempt_handling::execute( - Some(table_name), - "delete_table_file", - format!( - "Can not delete blob file: {}/{}. Err: {:?}", - table_name, blob_name, err - ), - attempt_no, - ) - .await; - attempt_no += 1; - } -} diff --git a/src/persist_io/azure/delete_table_folder.rs b/src/persist_io/azure/delete_table_folder.rs deleted file mode 100644 index e05b6b8..0000000 --- a/src/persist_io/azure/delete_table_folder.rs +++ /dev/null @@ -1,18 +0,0 @@ -use my_azure_storage_sdk::{blob_container::BlobContainersApi, AzureStorageConnection}; - -pub async fn delete_table_folder(azure_connection: &AzureStorageConnection, table_name: &str) { - let mut attempt_no = 0; - while let Err(err) = azure_connection - .delete_container_if_exists(table_name) - .await - { - super::attempt_handling::execute( - Some(table_name), - "delete_container", - format!("Can not delete container: {}. Err: {:?}", table_name, err), - attempt_no, - ) - .await; - attempt_no += 1; - } -} diff --git a/src/persist_io/azure/get_list_of_files.rs b/src/persist_io/azure/get_list_of_files.rs deleted file mode 100644 index 32cfc79..0000000 --- a/src/persist_io/azure/get_list_of_files.rs +++ /dev/null @@ -1,65 +0,0 @@ -use my_azure_storage_sdk::{ - blob_container::BlobContainersApi, sdk_azure::blobs::AzureBlobsListReader, - AzureStorageConnection, AzureStorageConnectionData, -}; - -use crate::persist_io::persist_io_operations::TableListOfFilesUploader; - -pub async fn get_list_of_files( - azure_connection: &AzureStorageConnection, - table_name: &str, - uploader: &TTableListOfFilesUploader, -) { - match azure_connection { - AzureStorageConnection::AzureStorage(connection_data) => { - get_list_of_files_from_azure_blob_container(connection_data, table_name, uploader) - .await; - } - _ => { - let chunk = azure_connection - .get_list_of_blobs(table_name) - .await - .unwrap(); - - uploader.add_files(table_name, chunk).await; - uploader.set_files_list_is_loaded(table_name).await; - } - }; -} - -async fn get_list_of_files_from_azure_blob_container< - TTableListOfFilesUploader: TableListOfFilesUploader, ->( - connection: &AzureStorageConnectionData, - table_name: &str, - uploader: &TTableListOfFilesUploader, -) { - let mut attempt_no: u8 = 0; - let mut blobs_list_reader = AzureBlobsListReader::new(connection, table_name); - loop { - let next_result = blobs_list_reader.get_next().await; - match next_result { - Ok(chunk) => { - if let Some(chunk) = chunk { - uploader.add_files(table_name, chunk).await; - } else { - uploader.set_files_list_is_loaded(table_name).await; - return; - } - } - Err(err) => { - super::attempt_handling::execute( - Some(table_name), - "get_list_of_files_from_azure_blob_container", - format!( - "Can not get list of files from azure blob container:[{}]. Err: {:?}", - table_name, err - ), - attempt_no, - ) - .await; - attempt_no += 1; - } - } - } -} diff --git a/src/persist_io/azure/get_list_of_tables.rs b/src/persist_io/azure/get_list_of_tables.rs deleted file mode 100644 index cb12ed8..0000000 --- a/src/persist_io/azure/get_list_of_tables.rs +++ /dev/null @@ -1,48 +0,0 @@ -use my_azure_storage_sdk::blob_container::BlobContainersApi; -use my_azure_storage_sdk::AzureStorageConnectionData; - -use my_azure_storage_sdk::sdk_azure::containers::AzureContainersListReader; -use my_azure_storage_sdk::AzureStorageConnection; - -pub async fn get_list_of_tables(azure_connection: &AzureStorageConnection) -> Vec { - match azure_connection { - AzureStorageConnection::AzureStorage(connection_data) => { - return get_list_of_tables_from_azure_blob_container(connection_data).await - } - _ => azure_connection - .get_list_of_blob_containers() - .await - .unwrap(), - } -} - -async fn get_list_of_tables_from_azure_blob_container( - connection: &AzureStorageConnectionData, -) -> Vec { - let mut result = Vec::new(); - let mut attempt_no: u8 = 0; - let mut blobs_list_reader = AzureContainersListReader::new(connection); - loop { - let next_result = blobs_list_reader.get_next().await; - match next_result { - Ok(chunk) => { - if let Some(chunk) = chunk { - result.extend(chunk) - } else { - return result; - } - } - Err(err) => { - super::attempt_handling::execute( - None, - "get_list_of_tables_from_azure_blob_container", - format!("Can not get list of tables. Err: {:?}", err), - attempt_no, - ) - .await; - - attempt_no += 1; - } - } - } -} diff --git a/src/persist_io/azure/load_table_file.rs b/src/persist_io/azure/load_table_file.rs deleted file mode 100644 index efd6758..0000000 --- a/src/persist_io/azure/load_table_file.rs +++ /dev/null @@ -1,43 +0,0 @@ -use my_azure_storage_sdk::{blob::BlobApi, AzureStorageConnection, AzureStorageError}; - -pub async fn load_table_file( - azure_connection: &AzureStorageConnection, - table_name: &str, - blob_file: &str, -) -> Option> { - let mut attempt_no = 0; - - loop { - let result = azure_connection.download_blob(table_name, blob_file).await; - - match result { - Ok(result) => { - return Some(result); - } - Err(err) => { - if let AzureStorageError::BlobNotFound = &err { - return None; - } - - super::attempt_handling::execute( - Some(table_name), - "load_table_file", - format!( - "Can not download blob {}/{}. Err: {:?}", - table_name, blob_file, err - ), - attempt_no, - ) - .await; - - if let AzureStorageError::InvalidResourceName = &err { - panic!( - "Can not download blob {}/{}. Reason: {:?}", - table_name, blob_file, err - ) - } - attempt_no += 1; - } - } - } -} diff --git a/src/persist_io/azure/mod.rs b/src/persist_io/azure/mod.rs deleted file mode 100644 index 43fc532..0000000 --- a/src/persist_io/azure/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -mod attempt_handling; -mod create_table; -mod delete_table_file; -mod delete_table_folder; -mod get_list_of_files; -mod get_list_of_tables; -mod load_table_file; -mod save_table_file; -pub use create_table::create_table; -pub use delete_table_file::delete_table_file; -pub use delete_table_folder::delete_table_folder; -pub use get_list_of_files::get_list_of_files; -pub use get_list_of_tables::get_list_of_tables; -pub use load_table_file::load_table_file; -pub use save_table_file::save_table_file; diff --git a/src/persist_io/azure/save_attributes.rs b/src/persist_io/azure/save_attributes.rs deleted file mode 100644 index 95d9f05..0000000 --- a/src/persist_io/azure/save_attributes.rs +++ /dev/null @@ -1,34 +0,0 @@ -use my_azure_storage_sdk::{block_blob::BlockBlobApi, AzureStorageConnection, AzureStorageError}; - -use crate::db::DbTableAttributesSnapshot; - -pub async fn save_attributes( - azure_connection: &AzureStorageConnection, - table_name: &str, - attributes: &DbTableAttributesSnapshot, -) -> Result<(), AzureStorageError> { - let contract = table_attrs::TableMetadataFileContract { - persist: attributes.persist, - max_partitions_amount: attributes.max_partitions_amount, - }; - - let serialize_result = serde_json::to_vec(&contract); - - match serialize_result { - Ok(json) => { - azure_connection - .upload(table_name, table_attrs::METADATA_FILE_NAME, json) - .await?; - - return Ok(()); - } - Err(err) => { - let msg = format!( - "Could not serialize table attributes to save it to the table. {}", - err - ); - - return Err(AzureStorageError::UnknownError { msg }); - } - }; -} diff --git a/src/persist_io/azure/save_table_file.rs b/src/persist_io/azure/save_table_file.rs deleted file mode 100644 index 2c2b56b..0000000 --- a/src/persist_io/azure/save_table_file.rs +++ /dev/null @@ -1,28 +0,0 @@ -use my_azure_storage_sdk::{block_blob::BlockBlobApi, AzureStorageConnection}; - -pub async fn save_table_file( - azure_connection: &AzureStorageConnection, - table_name: &str, - blob_name: &str, - content: Vec, -) { - let mut attempt_no = 0; - - while let Err(err) = azure_connection - .upload_block_blob(table_name, blob_name, content.to_vec()) - .await - { - super::attempt_handling::execute( - Some(table_name), - "save_table_file", - format!( - "Can not save blob {}/{}. Err: {:?}", - table_name, blob_name, err - ), - attempt_no, - ) - .await; - - attempt_no += 1; - } -} diff --git a/src/persist_io/mod.rs b/src/persist_io/mod.rs deleted file mode 100644 index 66a2a8b..0000000 --- a/src/persist_io/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -mod azure; -mod persist_io_operations; -mod sqlite; -mod table_file; -pub use persist_io_operations::{PersistIoOperations, TableListOfFilesUploader}; -pub use table_file::TableFile; - -pub const TABLE_METADATA_FILE_NAME: &str = ".metadata"; diff --git a/src/persist_io/persist_io_operations.rs b/src/persist_io/persist_io_operations.rs deleted file mode 100644 index c13f274..0000000 --- a/src/persist_io/persist_io_operations.rs +++ /dev/null @@ -1,229 +0,0 @@ -use std::{collections::BTreeMap, sync::Arc}; - -use my_azure_storage_sdk::AzureStorageConnection; - -use my_no_sql_server_core::db_snapshots::DbTableSnapshot; -use tokio::sync::Mutex; - -use crate::{persist_io::TableFile, sqlite_repo::SqlLiteRepo}; - -use super::{sqlite::InitContainer, TABLE_METADATA_FILE_NAME}; - -pub enum PersistIoOperations { - AzureConnection(Arc), - SqLite { - repo: SqlLiteRepo, - init_container: Mutex, - }, -} - -#[async_trait::async_trait] -pub trait TableListOfFilesUploader { - async fn add_files(&self, table_name: &str, files: Vec); - async fn set_files_list_is_loaded(&self, table_name: &str); -} - -impl PersistIoOperations { - pub fn as_azure_connection(azure_connection: Arc) -> Self { - Self::AzureConnection(azure_connection) - } - - pub fn as_sqlite(repo: SqlLiteRepo) -> Self { - Self::SqLite { - repo, - init_container: Mutex::new(InitContainer::new()), - } - } - - pub fn is_sqlite(&self) -> bool { - matches!(self, Self::SqLite { .. }) - } - - pub async fn get_list_of_tables(&self) -> Vec { - match self { - Self::SqLite { - repo, - init_container, - } => { - let tables = repo.get_files().await; - let mut read_access = init_container.lock().await; - read_access.init(tables); - read_access.get_list_of_tables() - } - Self::AzureConnection(azure_connection) => { - super::azure::get_list_of_tables(azure_connection.as_ref()).await - } - } - } - - pub async fn get_table_files( - &self, - table_name: &str, - uploader: &TTableListOfFilesUploader, - ) { - match self { - Self::SqLite { - repo: _, - init_container, - } => { - let init_container = init_container.lock().await; - let files = init_container.get_file_names(table_name); - uploader.add_files(table_name, files).await; - uploader.set_files_list_is_loaded(table_name).await; - } - Self::AzureConnection(azure_connection) => { - super::azure::get_list_of_files(azure_connection.as_ref(), table_name, uploader) - .await; - } - } - } - - pub async fn get_table_files_as_list( - &self, - table_name: &str, - ) -> Option>> { - match self { - Self::SqLite { - repo: _, - init_container, - } => { - let mut write_access = init_container.lock().await; - - write_access.get_files_by_table(table_name) - } - Self::AzureConnection(_) => { - panic!("Files or Microsoft Azure Init is not supported"); - } - } - } - - pub async fn create_table_folder(&self, table_name: &str) { - match self { - Self::SqLite { - repo: _, - init_container: _, - } => {} - Self::AzureConnection(azure_connection) => { - super::azure::create_table(azure_connection.as_ref(), table_name).await; - } - } - } - - pub async fn save_table_file( - &self, - table_name: &str, - table_file: &TableFile, - content: Vec, - ) { - match self { - Self::SqLite { - repo, - init_container: _, - } => { - repo.save_file( - table_name, - table_file.get_file_name().as_str(), - String::from_utf8(content).unwrap(), - ) - .await; - //super::sqlite::save_table_file(repo, table_name, table_file, content).await; - } - Self::AzureConnection(azure_connection) => { - super::azure::save_table_file( - azure_connection.as_ref(), - table_name, - table_file.get_file_name().as_str(), - content, - ) - .await; - } - } - /* - super::with_retries::save_table_file( - self.azure_connection.as_ref(), - table_name, - table_file.get_file_name().as_str(), - content, - ) - .await; - */ - } - - pub async fn delete_table_file(&self, table_name: &str, table_file: &TableFile) { - match self { - PersistIoOperations::AzureConnection(azure_connection) => { - super::azure::delete_table_file( - azure_connection.as_ref(), - table_name, - table_file.get_file_name().as_str(), - ) - .await; - } - PersistIoOperations::SqLite { - repo, - init_container: _, - } => match table_file { - TableFile::TableAttributes => { - repo.delete_file(table_name, TABLE_METADATA_FILE_NAME).await; - } - TableFile::DbPartition(partition_key) => { - repo.delete_file(table_name, partition_key.as_str()).await; - } - }, - } - } - - pub async fn delete_table_folder(&self, table_name: &str) { - match self { - PersistIoOperations::AzureConnection(azure_connection) => { - super::azure::delete_table_folder(azure_connection.as_ref(), table_name).await; - } - PersistIoOperations::SqLite { - repo: _, - init_container: _, - } => {} - } - } - - pub async fn load_table_file( - &self, - table_name: &str, - table_file: &TableFile, - ) -> Option> { - match self { - PersistIoOperations::AzureConnection(azure_connection) => { - super::azure::load_table_file( - azure_connection, - table_name, - table_file.get_file_name().as_str(), - ) - .await - } - PersistIoOperations::SqLite { - repo: _, - init_container, - } => { - let mut read_access = init_container.lock().await; - return read_access.get_file(table_name, table_file.get_file_name().as_str()); - } - } - } - - pub async fn init_table_from_other_source( - &self, - table_name: &str, - db_table_snapshot: DbTableSnapshot, - ) { - match self { - PersistIoOperations::AzureConnection(_) => { - panic!("Files or Microsoft Azure Init is not supported"); - } - PersistIoOperations::SqLite { - repo, - init_container: _, - } => { - super::sqlite::init_new_instance_table(repo, table_name, db_table_snapshot).await; - } - } - } -} diff --git a/src/persist_io/sqlite/init_container.rs b/src/persist_io/sqlite/init_container.rs deleted file mode 100644 index 569cfb9..0000000 --- a/src/persist_io/sqlite/init_container.rs +++ /dev/null @@ -1,74 +0,0 @@ -use std::collections::BTreeMap; - -use crate::{persist_io::TABLE_METADATA_FILE_NAME, sqlite_repo::MyNoSqlFileDto}; - -#[derive(Default)] -pub struct InitTableFiles { - files: BTreeMap>, -} - -impl InitTableFiles { - pub fn can_be_gc(&self) -> bool { - self.files.is_empty() - } -} - -pub struct InitContainer { - by_table: BTreeMap, -} - -impl InitContainer { - pub fn new() -> Self { - Self { - by_table: BTreeMap::new(), - } - } - - pub fn init(&mut self, dto_rows: Vec) { - for dto in dto_rows { - if let Some(table) = self.by_table.get_mut(&dto.table_name) { - table.files.insert(dto.file_name, dto.content.into_bytes()); - } else { - let mut table_files = InitTableFiles::default(); - table_files - .files - .insert(dto.file_name, dto.content.into_bytes()); - self.by_table.insert(dto.table_name, table_files); - } - } - } - - pub fn get_file_names(&self, table_name: &str) -> Vec { - let mut result = Vec::new(); - - result.push(TABLE_METADATA_FILE_NAME.to_string()); - - if let Some(table_data) = self.by_table.get(table_name) { - for pk in table_data.files.keys() { - result.push(pk.to_string()); - } - } - - result - } - - pub fn get_list_of_tables(&self) -> Vec { - self.by_table.keys().map(|itm| itm.to_string()).collect() - } - - pub fn get_file(&mut self, table_name: &str, partition_key: &str) -> Option> { - let table_data = self.by_table.get_mut(table_name)?; - - let result = table_data.files.remove(partition_key); - - if table_data.can_be_gc() { - self.by_table.remove(table_name); - } - - result - } - - pub fn get_files_by_table(&mut self, table_name: &str) -> Option>> { - self.by_table.remove(table_name).map(|itm| itm.files) - } -} diff --git a/src/persist_io/sqlite/init_new_instance_table.rs b/src/persist_io/sqlite/init_new_instance_table.rs deleted file mode 100644 index 4b9eab7..0000000 --- a/src/persist_io/sqlite/init_new_instance_table.rs +++ /dev/null @@ -1,43 +0,0 @@ -use my_no_sql_server_core::db_snapshots::DbTableSnapshot; - -use crate::{ - persist_io::TableFile, - persist_operations::serializers::TableMetadataFileContract, - sqlite_repo::{MyNoSqlFileDto, SqlLiteRepo}, -}; - -pub async fn init_new_instance_table( - repo: &SqlLiteRepo, - table_name: &str, - db_table_snapshot: DbTableSnapshot, -) { - let mut items_to_insert = Vec::new(); - let table_attributes: TableMetadataFileContract = (&db_table_snapshot.attr).into(); - - items_to_insert.push(MyNoSqlFileDto { - table_name: table_name.to_string(), - file_name: TableFile::TableAttributes - .get_file_name() - .as_str() - .to_string(), - content: String::from_utf8(table_attributes.to_vec()).unwrap(), - }); - - for partition in db_table_snapshot.by_partition { - let content = partition.db_rows_snapshot.as_json_array(); - let dto = MyNoSqlFileDto { - table_name: table_name.to_string(), - file_name: TableFile::DbPartition(partition.partition_key) - .get_file_name() - .as_str() - .to_string(), - content: String::from_utf8(content.build()).unwrap(), - }; - - items_to_insert.push(dto); - } - - for chunk in items_to_insert.chunks(1000) { - repo.save_files(chunk).await; - } -} diff --git a/src/persist_io/sqlite/mod.rs b/src/persist_io/sqlite/mod.rs deleted file mode 100644 index 2b0be1e..0000000 --- a/src/persist_io/sqlite/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -//mod save_table_file; -//pub use save_table_file::*; - -//mod get_list_of_files; -//pub use get_list_of_files::*; -mod init_container; -pub use init_container::*; -mod init_new_instance_table; -pub use init_new_instance_table::*; diff --git a/src/persist_io/table_file.rs b/src/persist_io/table_file.rs deleted file mode 100644 index 1aed4a1..0000000 --- a/src/persist_io/table_file.rs +++ /dev/null @@ -1,81 +0,0 @@ -use my_no_sql_sdk::core::db::PartitionKey; -use my_no_sql_sdk::core::rust_extensions::base64::FromBase64; - -use crate::persist_io::TABLE_METADATA_FILE_NAME; - -pub enum TableFile { - TableAttributes, - DbPartition(PartitionKey), -} - -pub struct TableFileName<'s> { - as_str: Option<&'s str>, - as_string: Option, -} - -impl<'s> TableFileName<'s> { - pub fn new(as_str: &'s str) -> Self { - Self { - as_str: Some(as_str), - as_string: None, - } - } - - pub fn new_as_string(as_string: String) -> Self { - Self { - as_string: Some(as_string), - as_str: None, - } - } - - pub fn as_str(&'s self) -> &'s str { - if let Some(as_str) = self.as_str { - return as_str; - } - - if let Some(as_string) = &self.as_string { - return as_string; - } - - panic!("TableFileName is not initialized properly"); - } -} - -impl TableFile { - pub fn from_file_name(file_name: &str) -> Result { - if file_name == TABLE_METADATA_FILE_NAME { - return Ok(Self::TableAttributes); - } - - let partition_key = file_name.from_base64(); - - if partition_key.is_err() { - return Err(format!( - "Can not decode filename: {}. Err:{:?}", - file_name, - partition_key.err() - )); - } - - let partition_key = partition_key.unwrap(); - - match String::from_utf8(partition_key) { - Ok(result) => Ok(Self::DbPartition(result.into())), - Err(err) => Err(format!( - "Can not decode filename: {}. Err:{:?}", - file_name, err - )), - } - } - pub fn get_file_name(&self) -> TableFileName { - match self { - TableFile::TableAttributes => TableFileName::new(TABLE_METADATA_FILE_NAME), - TableFile::DbPartition(partition_key) => { - use base64::Engine; - let encoded = base64::engine::general_purpose::STANDARD - .encode(partition_key.as_str().as_bytes()); - TableFileName::new_as_string(encoded) - } - } - } -} diff --git a/src/persist_markers/mod.rs b/src/persist_markers/mod.rs new file mode 100644 index 0000000..fbc37c8 --- /dev/null +++ b/src/persist_markers/mod.rs @@ -0,0 +1,15 @@ +mod persist_markers; +mod persist_partition_marker; +pub use persist_markers::*; +pub use persist_partition_marker::*; +//mod persist_markers_back; +//pub use persist_markers_back::*; +mod persist_row_marker; +pub use persist_row_marker::*; +mod persist_task; +pub use persist_task::*; +mod persist_by_table_item; +mod persist_markers_inner; +pub use persist_by_table_item::*; +mod persist_metrics; +pub use persist_metrics::*; diff --git a/src/persist_markers/persist_by_table_item.rs b/src/persist_markers/persist_by_table_item.rs new file mode 100644 index 0000000..b32834e --- /dev/null +++ b/src/persist_markers/persist_by_table_item.rs @@ -0,0 +1,170 @@ +use std::sync::Arc; + +use my_no_sql_sdk::core::db::{DbRow, DbTableName, PartitionKey}; +use my_no_sql_server_core::rust_extensions::{date_time::DateTimeAsMicroseconds, sorted_vec::*}; + +use super::{PersistMetrics, PersistPartitionMarker, PersistTask}; + +pub struct PersistByTableItem { + pub table_name: DbTableName, + pub persist_whole_table_content: Option, + pub persist_table_attributes: Option, + pub persist_partitions: SortedVecWithStrKey, + pub metrics: PersistMetrics, +} + +impl PersistByTableItem { + pub fn new(table_name: DbTableName) -> Self { + Self { + table_name, + persist_whole_table_content: None, + persist_table_attributes: None, + persist_partitions: SortedVecWithStrKey::new(), + metrics: PersistMetrics::default(), + } + } + + fn get_or_create_partition_item( + &mut self, + partition_key: &PartitionKey, + ) -> &mut PersistPartitionMarker { + let index = match self + .persist_partitions + .insert_or_if_not_exists(partition_key.as_str()) + { + InsertIfNotExists::Insert(insert_entity) => { + let index = insert_entity.index; + let item = PersistPartitionMarker::new(partition_key.clone()); + insert_entity.insert(item); + index + } + InsertIfNotExists::Exists(index) => index, + }; + + self.persist_partitions.get_by_index_mut(index).unwrap() + } + + pub fn persist_rows<'s>( + &mut self, + partition_key: &PartitionKey, + moment: DateTimeAsMicroseconds, + db_rows: impl Iterator>, + ) { + let by_partition = self.get_or_create_partition_item(partition_key); + + for db_row in db_rows { + by_partition.persist_row(db_row, moment); + } + } + + pub fn persist_whole_partition( + &mut self, + partition_key: &PartitionKey, + persist_moment: DateTimeAsMicroseconds, + ) { + let by_partition = self.get_or_create_partition_item(partition_key); + + match by_partition.persist_whole_partition { + Some(current_moment) => { + if current_moment > persist_moment { + by_partition.persist_whole_partition = Some(persist_moment); + } + } + None => { + by_partition.persist_whole_partition = Some(persist_moment); + } + } + } + + fn get_partition_to_sync(&self, now: Option) -> Option { + for partition in self.persist_partitions.iter() { + if let Some(persist_moment) = partition.persist_whole_partition { + if let Some(now) = now { + if persist_moment < now { + return Some(partition.partition_key.clone()); + } + } else { + return Some(partition.partition_key.clone()); + } + } + } + + None + } + + pub fn get_persist_task(&self, now: Option) -> Option { + if let Some(persist_moment) = self.persist_table_attributes { + if let Some(now) = now { + if persist_moment.unix_microseconds < now.unix_microseconds { + return Some(PersistTask::SaveTableAttributes(self.table_name.clone())); + } + } else { + return Some(PersistTask::SaveTableAttributes(self.table_name.clone())); + } + } + + if let Some(persist_moment) = self.persist_whole_table_content { + if let Some(now) = now { + if persist_moment.unix_microseconds < now.unix_microseconds { + return Some(PersistTask::SyncTable(self.table_name.clone())); + } + } else { + return Some(PersistTask::SyncTable(self.table_name.clone())); + } + } + + if let Some(partition_key) = self.get_partition_to_sync(now) { + return Some(PersistTask::SyncPartition { + table_name: self.table_name.clone(), + partition_key: partition_key, + }); + } + + let mut jobs = Vec::new(); + + for partition in self.persist_partitions.iter() { + let items = partition.get_row_to_sync(now); + + if items.len() > 0 { + jobs.push(super::SyncRowJobDescription { + partition_key: partition.partition_key.clone(), + items, + }); + } + } + + if jobs.len() > 0 { + return Some(PersistTask::SyncRows { + table_name: self.table_name.clone(), + jobs, + }); + } + + None + } + + pub fn clean_when_synching_whole_table(&mut self) { + self.persist_whole_table_content = None; + self.persist_partitions.clear(None); + } + + pub fn clean_when_synching_whole_partition(&mut self, partition_key: &PartitionKey) { + self.persist_partitions + .get_mut(partition_key.as_str()) + .unwrap() + .clean_when_synching_whole_partition(); + } + + pub fn clean_when_syncing_rows(&mut self, partition_key: &PartitionKey, rows: &[Arc]) { + self.persist_partitions + .get_mut(partition_key.as_str()) + .unwrap() + .clean_when_syncing_rows(rows); + } +} + +impl EntityWithStrKey for PersistByTableItem { + fn get_key(&self) -> &str { + self.table_name.as_str() + } +} diff --git a/src/persist_markers/persist_markers.rs b/src/persist_markers/persist_markers.rs new file mode 100644 index 0000000..8d55c7a --- /dev/null +++ b/src/persist_markers/persist_markers.rs @@ -0,0 +1,274 @@ +use std::sync::Arc; +use std::time::Duration; + +use my_no_sql_sdk::core::db::{DbRow, DbTableName, PartitionKey}; +use my_no_sql_sdk::core::rust_extensions::date_time::DateTimeAsMicroseconds; +use tokio::sync::Mutex; + +use super::persist_markers_inner::PersistMarkersInner; +use super::{PersistMetrics, PersistTask}; + +pub struct PersistMarkers { + inner: Mutex, +} + +impl PersistMarkers { + pub fn new() -> Self { + Self { + inner: Mutex::new(PersistMarkersInner::new()), + } + } + + pub async fn delete_db_rows( + &self, + table_name: &DbTableName, + partition_key: &PartitionKey, + persist_moment: DateTimeAsMicroseconds, + rows_to_delete: impl Iterator>, + ) { + let mut inner = self.inner.lock().await; + inner.persist_rows(table_name, partition_key, persist_moment, rows_to_delete); + } + + pub async fn persist_rows( + &self, + table_name: &DbTableName, + partition_key: &PartitionKey, + persist_moment: DateTimeAsMicroseconds, + rows_to_persist: impl Iterator>, + ) { + let mut inner = self.inner.lock().await; + inner.persist_rows(table_name, partition_key, persist_moment, rows_to_persist); + } + + pub async fn persist_partition( + &self, + table_name: &DbTableName, + partition_key: &PartitionKey, + persist_moment: DateTimeAsMicroseconds, + ) { + let mut inner = self.inner.lock().await; + inner.persist_whole_partition(table_name, partition_key, persist_moment); + } + + pub async fn persist_table_content( + &self, + table_name: &DbTableName, + persist_moment: DateTimeAsMicroseconds, + ) { + let mut inner = self.inner.lock().await; + inner.persist_table_content(table_name, persist_moment); + } + + pub async fn persist_table_attributes( + &self, + table_name: &DbTableName, + persist_moment: DateTimeAsMicroseconds, + ) { + let mut inner = self.inner.lock().await; + inner.persist_table_attributes(table_name, persist_moment); + } + + pub async fn get_persist_task( + &self, + now: Option, + ) -> Option { + let mut inner = self.inner.lock().await; + + let result = inner.get_persist_task(now); + + if let Some(result) = result.as_ref() { + match result { + PersistTask::SaveTableAttributes(db_table_name) => { + inner + .get_by_table_mut(db_table_name) + .unwrap() + .persist_table_attributes = None; + } + PersistTask::SyncTable(db_table_name) => { + inner + .get_by_table_mut(db_table_name) + .unwrap() + .clean_when_synching_whole_table(); + } + + PersistTask::SyncPartition { + table_name, + partition_key, + } => { + let by_table = inner.get_by_table_mut(table_name).unwrap(); + by_table.clean_when_synching_whole_partition(partition_key); + } + PersistTask::SyncRows { table_name, jobs } => { + let by_table = inner.get_by_table_mut(table_name).unwrap(); + for job in jobs { + by_table.clean_when_syncing_rows(&job.partition_key, &job.items); + } + } + } + } + + result + } + + pub async fn set_last_persist_time( + &self, + table_name: &DbTableName, + now: DateTimeAsMicroseconds, + duration: Duration, + ) { + let mut inner = self.inner.lock().await; + inner.set_last_persist_time(table_name, now, duration); + } + + pub async fn get_persist_metrics(&self, table_name: &str) -> PersistMetrics { + let inner = self.inner.lock().await; + match inner.get_by_table(table_name) { + Some(metrics) => metrics.metrics.clone(), + None => PersistMetrics::default(), + } + } + + /* + pub async fn persist_partition( + &self, + db_table: &DbTable, + partition_key: &impl PartitionKeyParameter, + sync_moment: DateTimeAsMicroseconds, + ) { + if !db_table.attributes.persist { + return; + } + + let mut write_access = self.by_table.lock().await; + + match write_access.insert_or_update(db_table.name.as_str()) { + my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Insert( + entry, + ) => { + let mut item = PersistByTableItem { + table_name: db_table.name.clone(), + data: TablePersistData::new(), + }; + + item.data + .data_to_persist + .mark_partition_to_persist(partition_key, sync_moment); + + entry.insert(item); + } + my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Update( + entry, + ) => { + entry + .item + .data + .data_to_persist + .mark_partition_to_persist(partition_key, sync_moment); + } + } + } + + pub async fn persist_table(&self, db_table: &DbTable, sync_moment: DateTimeAsMicroseconds) { + if !db_table.attributes.persist { + return; + } + + let mut write_access = self.by_table.lock().await; + + match write_access.insert_or_update(&db_table.name) { + my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Insert( + entry, + ) => { + let mut item = PersistByTableItem { + table_name: db_table.name.to_string(), + data: TablePersistData::new(), + }; + + item.data.data_to_persist.mark_table_to_persist(sync_moment); + + entry.insert(item); + } + my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Update( + entry, + ) => { + entry + .item + .data + .data_to_persist + .mark_table_to_persist(sync_moment); + } + } + } + + pub async fn persist_table_attrs(&self, db_table: &DbTable) { + if !db_table.attributes.persist { + return; + } + + let mut write_access = self.by_table.lock().await; + + match write_access.insert_or_update(&db_table.name) { + my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Insert( + entry, + ) => { + let mut item = PersistByTableItem { + table_name: db_table.name.to_string(), + data: TablePersistData::new(), + }; + + item.data.data_to_persist.mark_persist_attrs(); + + entry.insert(item); + } + my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Update( + entry, + ) => { + entry.item.data.data_to_persist.mark_persist_attrs(); + } + } + } + + pub async fn get_job_to_persist( + &self, + table_name: &str, + now: DateTimeAsMicroseconds, + is_shutting_down: bool, + ) -> Option { + let mut write_access = self.by_table.lock().await; + + let item = write_access.get_mut(table_name)?; + + item.data + .data_to_persist + .get_what_to_persist(now, is_shutting_down) + } + + pub async fn set_persisted(&self, table_name: &str, duration: Duration) { + let mut write_access = self.by_table.lock().await; + + if let Some(item) = write_access.get_mut(table_name) { + item.data.add_persist_duration(duration); + } + } + + pub async fn get_persist_metrics(&self, table_name: &str) -> PersistMetrics { + let read_access = self.by_table.lock().await; + + match read_access.get(table_name) { + Some(result) => PersistMetrics { + last_persist_time: result.data.last_persist_time.clone(), + next_persist_time: result.data.data_to_persist.get_next_persist_time(), + persist_amount: result.data.data_to_persist.get_persist_amount(), + last_persist_duration: result.data.persist_duration.clone(), + }, + None => PersistMetrics { + last_persist_time: None, + next_persist_time: None, + persist_amount: 0, + last_persist_duration: vec![], + }, + } + } + */ +} diff --git a/src/persist_markers/persist_markers_inner.rs b/src/persist_markers/persist_markers_inner.rs new file mode 100644 index 0000000..736564c --- /dev/null +++ b/src/persist_markers/persist_markers_inner.rs @@ -0,0 +1,126 @@ +use std::{sync::Arc, time::Duration}; + +use my_no_sql_sdk::core::db::{DbRow, DbTableName, PartitionKey}; +use my_no_sql_server_core::rust_extensions::{date_time::DateTimeAsMicroseconds, sorted_vec::*}; + +use super::{PersistByTableItem, PersistTask}; + +pub struct PersistMarkersInner { + items: SortedVecWithStrKey, +} + +impl PersistMarkersInner { + pub fn new() -> Self { + Self { + items: SortedVecWithStrKey::new(), + } + } + + fn get_item_mut(&mut self, table_name: &DbTableName) -> &mut PersistByTableItem { + let index = match self.items.insert_or_if_not_exists(table_name.as_str()) { + InsertIfNotExists::Insert(insert_entity) => { + let index = insert_entity.index; + let item = PersistByTableItem::new(table_name.clone()); + insert_entity.insert(item); + index + } + InsertIfNotExists::Exists(index) => index, + }; + + self.items.get_by_index_mut(index).unwrap() + } + + pub fn persist_table_content( + &mut self, + table_name: &DbTableName, + persist_moment: DateTimeAsMicroseconds, + ) { + let item = self.get_item_mut(table_name); + + match item.persist_whole_table_content { + Some(date_time) => { + if persist_moment < date_time { + item.persist_whole_table_content = Some(persist_moment); + } + } + None => { + item.persist_whole_table_content = Some(persist_moment); + } + } + } + + pub fn persist_table_attributes( + &mut self, + table_name: &DbTableName, + persist_moment: DateTimeAsMicroseconds, + ) { + let item = self.get_item_mut(table_name); + + match item.persist_table_attributes { + Some(date_time) => { + if persist_moment < date_time { + item.persist_table_attributes = Some(persist_moment); + } + } + None => { + item.persist_table_attributes = Some(persist_moment); + } + } + } + + pub fn persist_rows<'s>( + &mut self, + table_name: &DbTableName, + partition_key: &PartitionKey, + moment: DateTimeAsMicroseconds, + rows_to_persist: impl Iterator>, + ) { + let item = self.get_item_mut(table_name); + + item.persist_rows(partition_key, moment, rows_to_persist); + } + + pub fn persist_whole_partition( + &mut self, + table_name: &DbTableName, + partition_key: &PartitionKey, + persist_moment: DateTimeAsMicroseconds, + ) { + let item = self.get_item_mut(table_name); + + item.persist_whole_partition(partition_key, persist_moment); + } + + pub fn get_by_table_mut( + &mut self, + table_name: &DbTableName, + ) -> Option<&mut PersistByTableItem> { + self.items.get_mut(table_name.as_str()) + } + + pub fn get_by_table(&self, table_name: &str) -> Option<&PersistByTableItem> { + self.items.get(table_name) + } + + pub fn get_persist_task(&self, now: Option) -> Option { + for itm in self.items.iter() { + let persist_task = itm.get_persist_task(now); + + if persist_task.is_some() { + return persist_task; + } + } + + None + } + + pub fn set_last_persist_time( + &mut self, + table_name: &DbTableName, + moment: DateTimeAsMicroseconds, + duration: Duration, + ) { + let item = self.get_item_mut(table_name); + item.metrics.update(moment, duration); + } +} diff --git a/src/persist_markers/persist_metrics.rs b/src/persist_markers/persist_metrics.rs new file mode 100644 index 0000000..1496eb2 --- /dev/null +++ b/src/persist_markers/persist_metrics.rs @@ -0,0 +1,23 @@ +use std::time::Duration; + +use my_no_sql_server_core::rust_extensions::date_time::DateTimeAsMicroseconds; + +#[derive(Debug, Default, Clone)] +pub struct PersistMetrics { + pub last_persist_time: Option, + pub next_persist_time: Option, + pub persist_amount: usize, + pub last_persist_duration: Vec, +} + +impl PersistMetrics { + pub fn update(&mut self, last_persist_time: DateTimeAsMicroseconds, duration: Duration) { + self.last_persist_time = Some(last_persist_time); + self.last_persist_duration + .push(duration.as_micros() as usize); + + if self.last_persist_duration.len() > 100 { + self.last_persist_duration.remove(0); + } + } +} diff --git a/src/persist/partition_persist_marker.rs b/src/persist_markers/persist_partition_marker.rs similarity index 71% rename from src/persist/partition_persist_marker.rs rename to src/persist_markers/persist_partition_marker.rs index dd68664..5325eb5 100644 --- a/src/persist/partition_persist_marker.rs +++ b/src/persist_markers/persist_partition_marker.rs @@ -1,63 +1,76 @@ -use my_no_sql_sdk::core::db::{PartitionKey, PartitionKeyParameter}; -use my_no_sql_sdk::core::rust_extensions::{ - date_time::DateTimeAsMicroseconds, - sorted_vec::{EntityWithStrKey, SortedVecWithStrKey}, -}; +use std::sync::Arc; -pub enum PersistResult { - PersistAttrs, - PersistTable, - PersistPartition(PartitionKey), -} +use my_no_sql_sdk::core::db::{DbRow, PartitionKey}; +use my_no_sql_sdk::core::rust_extensions::date_time::DateTimeAsMicroseconds; +use my_no_sql_server_core::rust_extensions::sorted_vec::{ + EntityWithStrKey, InsertOrUpdateEntry, SortedVecWithStrKey, +}; -impl PersistResult { - #[cfg(test)] - pub fn is_table(&self) -> bool { - match self { - PersistResult::PersistAttrs => false, - PersistResult::PersistTable => true, - PersistResult::PersistPartition(_) => false, - } - } -} +use super::PersistRowMarker; -pub struct PartitionPersistMoment { +pub struct PersistPartitionMarker { pub partition_key: PartitionKey, - pub persist_moment: DateTimeAsMicroseconds, + pub persist_whole_partition: Option, + pub rows_to_persist: SortedVecWithStrKey, } -impl EntityWithStrKey for PartitionPersistMoment { +impl EntityWithStrKey for PersistPartitionMarker { fn get_key(&self) -> &str { self.partition_key.as_str() } } -pub struct PartitionPersistMarker { - pub persist_whole_table: Option, - pub partitions: SortedVecWithStrKey, - pub persist_attrs: bool, -} +impl PersistPartitionMarker { + pub fn new(partition_key: PartitionKey) -> Self { + Self { + partition_key, + persist_whole_partition: None, + rows_to_persist: SortedVecWithStrKey::new(), + } + } + + pub fn persist_row(&mut self, db_row: &Arc, persist_moment: DateTimeAsMicroseconds) { + match self.rows_to_persist.insert_or_update(db_row.get_row_key()) { + InsertOrUpdateEntry::Insert(insert_entity) => { + insert_entity.insert(PersistRowMarker::new(db_row.clone(), persist_moment)); + } + InsertOrUpdateEntry::Update(update_entry) => { + if update_entry.item.persist_moment > persist_moment { + update_entry.item.persist_moment = persist_moment; + } -impl PartitionPersistMarker { - pub fn get_persist_amount(&self) -> usize { - let mut result = if self.persist_attrs { 1 } else { 0 }; - result += self.partitions.len(); + update_entry.item.db_row = db_row.clone(); + } + } + } - if self.persist_whole_table.is_some() { - result += 1; - }; + pub fn get_row_to_sync(&self, now: Option) -> Vec> { + let mut result = Vec::new(); + for item in self.rows_to_persist.iter() { + if let Some(now) = now { + if item.persist_moment.unix_microseconds <= now.unix_microseconds { + result.push(item.db_row.clone()); + } + } else { + result.push(item.db_row.clone()); + } + } result } - pub fn new() -> Self { - Self { - persist_whole_table: None, - partitions: SortedVecWithStrKey::new(), - persist_attrs: false, + pub fn clean_when_synching_whole_partition(&mut self) { + self.persist_whole_partition = None; + self.rows_to_persist.clear(None); + } + + pub fn clean_when_syncing_rows(&mut self, rows: &[Arc]) { + for row in rows { + self.rows_to_persist.remove(row.get_row_key()); } } + /* pub fn mark_table_to_persist(&mut self, moment: DateTimeAsMicroseconds) { if self.persist_whole_table.is_none() { self.persist_whole_table = Some(moment); @@ -163,8 +176,10 @@ impl PartitionPersistMarker { } None } + */ } +/* #[cfg(test)] mod test { use super::*; @@ -220,3 +235,4 @@ mod test { assert_eq!(true, result.is_table()); } } + */ diff --git a/src/persist_markers/persist_row_marker.rs b/src/persist_markers/persist_row_marker.rs new file mode 100644 index 0000000..e36523d --- /dev/null +++ b/src/persist_markers/persist_row_marker.rs @@ -0,0 +1,26 @@ +use std::sync::Arc; + +use my_no_sql_sdk::core::db::DbRow; +use my_no_sql_server_core::rust_extensions::{ + date_time::DateTimeAsMicroseconds, sorted_vec::EntityWithStrKey, +}; + +pub struct PersistRowMarker { + pub db_row: Arc, + pub persist_moment: DateTimeAsMicroseconds, +} + +impl PersistRowMarker { + pub fn new(db_row: Arc, moment: DateTimeAsMicroseconds) -> Self { + Self { + db_row, + persist_moment: moment, + } + } +} + +impl EntityWithStrKey for PersistRowMarker { + fn get_key(&self) -> &str { + self.db_row.get_row_key() + } +} diff --git a/src/persist_markers/persist_task.rs b/src/persist_markers/persist_task.rs new file mode 100644 index 0000000..73bfb2d --- /dev/null +++ b/src/persist_markers/persist_task.rs @@ -0,0 +1,21 @@ +use std::sync::Arc; + +use my_no_sql_sdk::core::db::{DbRow, DbTableName, PartitionKey}; + +pub struct SyncRowJobDescription { + pub partition_key: PartitionKey, + pub items: Vec>, +} + +pub enum PersistTask { + SaveTableAttributes(DbTableName), + SyncTable(DbTableName), + SyncPartition { + table_name: DbTableName, + partition_key: PartitionKey, + }, + SyncRows { + table_name: DbTableName, + jobs: Vec, + }, +} diff --git a/src/persist_operations/blob_content_cache/blob_content_cache.rs b/src/persist_operations/blob_content_cache/blob_content_cache.rs deleted file mode 100644 index bfe5704..0000000 --- a/src/persist_operations/blob_content_cache/blob_content_cache.rs +++ /dev/null @@ -1,140 +0,0 @@ -use my_no_sql_sdk::core::db::{ - db_table_master_node::PartitionLastWriteMoment, DbTable, DbTableAttributes, -}; -use my_no_sql_sdk::core::rust_extensions::{ - date_time::DateTimeAsMicroseconds, sorted_vec::SortedVecWithStrKey, -}; -use my_no_sql_server_core::db_snapshots::DbPartitionSnapshot; -use tokio::sync::RwLock; - -use super::PersistedTableData; - -pub enum BlobPartitionUpdateTimeResult { - Ok(DateTimeAsMicroseconds), - TableNotFound, - PartitionNoFound, -} - -pub struct BlobContentCache { - pub data_by_table: RwLock>, -} - -impl BlobContentCache { - pub fn new() -> Self { - Self { - data_by_table: RwLock::new(SortedVecWithStrKey::new()), - } - } - - pub async fn has_table(&self, table_name: &str) -> bool { - let read_access = self.data_by_table.read().await; - read_access.contains(table_name) - } - - pub async fn init(&self, table_data: &DbTable) { - let data_to_insert = PersistedTableData::init(table_data); - let mut write_access = self.data_by_table.write().await; - write_access.insert_or_replace(data_to_insert); - } - - pub async fn create_table(&self, table_name: &str, attr: &DbTableAttributes) { - let data_to_insert = PersistedTableData::new(table_name.to_string(), attr.clone()); - let mut write_access = self.data_by_table.write().await; - write_access.insert_or_replace(data_to_insert); - } - - pub async fn update_table_attributes(&self, table_name: &str, attr: DbTableAttributes) { - let mut write_access = self.data_by_table.write().await; - - match write_access.insert_or_update(table_name) { - my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Insert( - entry, - ) => { - let table_data = PersistedTableData::new(table_name.to_string(), attr); - entry.insert(table_data); - } - my_no_sql_sdk::core::rust_extensions::sorted_vec::InsertOrUpdateEntry::Update( - entry, - ) => { - entry.item.attr = attr; - } - } - } - - pub async fn delete_table(&self, table_name: &str) { - let mut write_access = self.data_by_table.write().await; - write_access.remove(table_name); - } - - pub async fn delete_table_partition(&self, table_name: &str, partition_key: &str) { - let mut write_access = self.data_by_table.write().await; - - let table = write_access.get_mut(table_name); - - if let Some(table) = table { - table.partitions.remove(partition_key); - } - } - - pub async fn update_table_partition_snapshot_id( - &self, - table_name: &str, - db_partition_snapshot: DbPartitionSnapshot, - ) { - let mut write_access = self.data_by_table.write().await; - - let table = write_access.get_mut(table_name); - - if let Some(table) = table { - table - .partitions - .insert_or_replace(PartitionLastWriteMoment { - partition_key: db_partition_snapshot.partition_key, - last_write_moment: db_partition_snapshot.last_write_moment, - }); - } - } - - pub async fn get_snapshot( - &self, - table_name: &str, - ) -> Option> { - let read_access = self.data_by_table.read().await; - let table = read_access.get(table_name)?; - - let mut result = SortedVecWithStrKey::new(); - - for db_partition in table.partitions.iter() { - result.insert_or_replace(PartitionLastWriteMoment { - partition_key: db_partition.partition_key.clone(), - last_write_moment: db_partition.last_write_moment, - }); - } - - Some(result) - } - - pub async fn get( - &self, - table_name: &str, - partition_key: &str, - ) -> BlobPartitionUpdateTimeResult { - let read_access = self.data_by_table.read().await; - - let table = read_access.get(table_name); - - if table.is_none() { - return BlobPartitionUpdateTimeResult::TableNotFound; - } - - let table = table.unwrap(); - - let result = table.partitions.get(partition_key); - - if result.is_none() { - return BlobPartitionUpdateTimeResult::PartitionNoFound; - } - - BlobPartitionUpdateTimeResult::Ok(result.unwrap().last_write_moment) - } -} diff --git a/src/persist_operations/blob_content_cache/mod.rs b/src/persist_operations/blob_content_cache/mod.rs deleted file mode 100644 index 965ca33..0000000 --- a/src/persist_operations/blob_content_cache/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod blob_content_cache; -mod persisted_table_data; -pub use blob_content_cache::{BlobContentCache, BlobPartitionUpdateTimeResult}; -pub use persisted_table_data::PersistedTableData; diff --git a/src/persist_operations/blob_content_cache/persisted_table_data.rs b/src/persist_operations/blob_content_cache/persisted_table_data.rs deleted file mode 100644 index d3c90c8..0000000 --- a/src/persist_operations/blob_content_cache/persisted_table_data.rs +++ /dev/null @@ -1,34 +0,0 @@ -use my_no_sql_sdk::core::db::{ - db_table_master_node::PartitionLastWriteMoment, DbTable, DbTableAttributes, -}; -use my_no_sql_sdk::core::rust_extensions::sorted_vec::{EntityWithStrKey, SortedVecWithStrKey}; - -pub struct PersistedTableData { - pub table_name: String, - pub attr: DbTableAttributes, - pub partitions: SortedVecWithStrKey, -} - -impl EntityWithStrKey for PersistedTableData { - fn get_key(&self) -> &str { - self.table_name.as_str() - } -} - -impl PersistedTableData { - pub fn new(table_name: String, attr: DbTableAttributes) -> Self { - Self { - table_name, - attr, - partitions: SortedVecWithStrKey::new(), - } - } - - pub fn init(db_table: &DbTable) -> Self { - Self { - table_name: db_table.name.clone(), - attr: db_table.attributes.clone(), - partitions: db_table.get_partitions_last_write_moment(), - } - } -} diff --git a/src/persist_operations/data_initializer/from_other_instance/init_from_another_instance.rs b/src/persist_operations/data_initializer/from_other_instance/init_from_another_instance.rs deleted file mode 100644 index c8e9e55..0000000 --- a/src/persist_operations/data_initializer/from_other_instance/init_from_another_instance.rs +++ /dev/null @@ -1,67 +0,0 @@ -use std::{collections::BTreeMap, sync::Arc}; - -use my_no_sql_sdk::core::db::{DbPartition, DbTable}; -use my_no_sql_server_core::{rust_extensions::date_time::DateTimeAsMicroseconds, DbTableWrapper}; - -use crate::{ - app::AppContext, persist_io::TableFile, - persist_operations::serializers::TableMetadataFileContract, -}; - -pub async fn init_from_another_instance(app: &Arc, url: &str) { - let tables = super::load_tables(url).await; - - println!("Loaded {} tables from instance: {}", tables.len(), url); - - for table in tables { - println!("Initializing table {}", table.name); - app.persist_io.create_table_folder(&table.name).await; - - let table_file = TableFile::TableAttributes; - - let table_metadata_contract = TableMetadataFileContract { - persist: table.persist.unwrap_or(true), - max_partitions_amount: table.max_partitions_amount.map(|itm| itm as usize), - max_rows_per_partition_amount: table - .max_rows_per_partition_amount - .map(|itm| itm as usize), - created: Some(DateTimeAsMicroseconds::now().to_rfc3339()), - }; - - app.persist_io - .save_table_file(&table.name, &table_file, table_metadata_contract.to_vec()) - .await; - - let table_content = super::load_rows(url, &table.name).await; - - let mut by_partition: BTreeMap>> = - BTreeMap::new(); - - for row in table_content { - if let Some(db_partition) = by_partition.get_mut(row.get_partition_key()) { - db_partition.push(row); - } else { - by_partition.insert(row.get_partition_key().to_string(), vec![row]); - } - } - - let mut db_table = DbTable::new(table.name.to_string(), table_metadata_contract.into()); - - for (partition_key, rows) in by_partition { - let mut db_partition = DbPartition::new(partition_key); - db_partition.insert_or_replace_rows_bulk(rows.as_slice()); - db_table.init_partition(db_partition); - } - - let db_table = DbTableWrapper::new(db_table); - - let table_snapshot = db_table.get_table_snapshot().await; - - app.persist_io - .init_table_from_other_source(&table.name, table_snapshot) - .await; - - let mut table_access = app.db.tables.write().await; - table_access.insert(db_table.name.to_string(), db_table); - } -} diff --git a/src/persist_operations/data_initializer/from_other_instance/load_rows.rs b/src/persist_operations/data_initializer/from_other_instance/load_rows.rs deleted file mode 100644 index ecc253d..0000000 --- a/src/persist_operations/data_initializer/from_other_instance/load_rows.rs +++ /dev/null @@ -1,18 +0,0 @@ -use std::sync::Arc; - -use flurl::FlUrl; -use my_no_sql_sdk::core::{db::DbRow, db_json_entity::DbJsonEntity}; - -pub async fn load_rows(url: &str, table_name: &str) -> Vec> { - let mut response = FlUrl::new(url) - .append_path_segment("api") - .append_path_segment("Row") - .append_query_param("tableName", Some(table_name)) - .get() - .await - .unwrap(); - - let body = response.get_body_as_slice().await.unwrap(); - - DbJsonEntity::restore_as_vec(body).unwrap() -} diff --git a/src/persist_operations/data_initializer/from_other_instance/load_tables.rs b/src/persist_operations/data_initializer/from_other_instance/load_tables.rs deleted file mode 100644 index 29d0961..0000000 --- a/src/persist_operations/data_initializer/from_other_instance/load_tables.rs +++ /dev/null @@ -1,25 +0,0 @@ -use flurl::FlUrl; -use serde::*; - -#[derive(Debug, Serialize, Deserialize)] -pub struct TableMyNoSqlServerContract { - pub name: String, - pub persist: Option, - #[serde(rename = "maxPartitionsAmount")] - pub max_partitions_amount: Option, - pub max_rows_per_partition_amount: Option, -} - -pub async fn load_tables(url: &str) -> Vec { - let mut response = FlUrl::new(url) - .append_path_segment("api") - .append_path_segment("Tables") - .append_path_segment("List") - .get() - .await - .unwrap(); - - let body = response.get_body_as_slice().await.unwrap(); - - serde_json::from_slice(body).unwrap() -} diff --git a/src/persist_operations/data_initializer/load_table_files.rs b/src/persist_operations/data_initializer/load_table_files.rs deleted file mode 100644 index b12ffaa..0000000 --- a/src/persist_operations/data_initializer/load_table_files.rs +++ /dev/null @@ -1,62 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use crate::{app::AppContext, persist_io::TableFile}; - -use super::{load_tasks::NextFileToLoadResult, LoadedTableItem}; -use my_logger::LogEventCtx; - -pub async fn spawn(app: Arc) { - loop { - let next_file_to_load = app.init_state.get_next_file_to_load().await; - - match next_file_to_load { - NextFileToLoadResult::FileToLoad { - table_name, - file_name, - } => { - let table_file = TableFile::from_file_name(file_name.as_str()); - - if let Err(err) = table_file { - my_logger::LOGGER.write_error( - "init_tables".to_string(), - format!("Error loading table file {}: {}", file_name, err), - LogEventCtx::new() - .add("tableName", table_name) - .add("fileName", file_name), - ); - continue; - } - - let table_file = table_file.unwrap(); - - let content = app - .persist_io - .load_table_file(table_name.as_str(), &table_file) - .await; - - if let Some(content) = content.as_ref() { - match LoadedTableItem::new(&table_file, content) { - Ok(table_item) => { - app.init_state - .upload_table_file(table_name.as_str(), file_name, table_item) - .await; - } - Err(err) => { - my_logger::LOGGER.write_error( - "init_tables".to_string(), - format!("Error parsing table. Err: {}", err), - LogEventCtx::new() - .add("tableName", table_name) - .add("fileName", file_name), - ); - } - } - } - } - NextFileToLoadResult::NotReadyYet => { - tokio::time::sleep(Duration::from_secs(1)).await; - } - NextFileToLoadResult::NothingToLoad => return, - } - } -} diff --git a/src/persist_operations/data_initializer/load_tables.rs b/src/persist_operations/data_initializer/load_tables.rs deleted file mode 100644 index 930e8ca..0000000 --- a/src/persist_operations/data_initializer/load_tables.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::sync::Arc; - -use my_logger::LogEventCtx; -use my_no_sql_sdk::core::{ - db::{DbPartition, DbTable, DbTableAttributes}, - db_json_entity::DbJsonEntity, - rust_extensions::StopWatch, -}; -use my_no_sql_server_core::DbTableWrapper; - -use crate::{ - app::AppContext, persist_io::TableFile, - persist_operations::serializers::TableMetadataFileContract, -}; - -pub async fn load_tables(app: Arc) { - let mut sw = StopWatch::new(); - sw.start(); - if let Some(url) = app.settings.get_init_from_other_server_url() { - super::from_other_instance::init_from_another_instance(&app, url).await; - } else { - if app.persist_io.is_sqlite() { - init_as_sql_lite(&app).await; - } else { - init_from_storage(&app).await; - } - } - - app.states.set_initialized(); - - app.init_state.dispose().await; - - sw.pause(); - - my_logger::LOGGER.write_info( - "init_tables".to_string(), - format!("All tables initialized in {:?}", sw.duration()), - LogEventCtx::new(), - ); -} - -async fn init_as_sql_lite(app: &Arc) { - let table_names = app.persist_io.get_list_of_tables().await; - - println!("Got tables {:?}", table_names.len()); - - let mut loaded = 0; - for table_name in table_names { - if loaded % 10 == 0 { - println!("Loaded {} tables", loaded); - } - loaded += 1; - - let db_table = app - .persist_io - .get_table_files_as_list(table_name.as_str()) - .await; - - if db_table.is_none() { - panic!("Table {} is not found. Something went wrong", table_name); - } - - let mut table_attributes = None; - - let mut partitions = Vec::new(); - - for (file_name, file_content) in db_table.unwrap() { - let table_file = TableFile::from_file_name(file_name.as_str()).unwrap(); - - match table_file { - TableFile::TableAttributes => { - let metadata_file = TableMetadataFileContract::parse(file_content.as_slice()); - - let attr: DbTableAttributes = metadata_file.into(); - table_attributes = Some(attr); - } - TableFile::DbPartition(partition_key) => { - let db_rows = DbJsonEntity::restore_as_vec(file_content.as_slice()).unwrap(); - let mut restored_partition = DbPartition::new(partition_key); - restored_partition.insert_or_replace_rows_bulk(db_rows.as_slice()); - partitions.push(restored_partition); - } - } - } - - let mut db_table = DbTable::new(table_name, table_attributes.unwrap_or_default()); - - for db_partition in partitions { - db_table.init_partition(db_partition); - } - - let db_table = DbTableWrapper::new(db_table); - - let mut table_access = app.db.tables.write().await; - table_access.insert(db_table.name.to_string(), db_table); - - // crate::db_operations::write::table::init(app.as_ref(), db_table).await; - } -} - -async fn init_from_storage(app: &Arc) { - let table_names = app.persist_io.get_list_of_tables().await; - - app.init_state.init_table_names(table_names.clone()).await; - - tokio::spawn(super::table_list_of_files_loader(app.clone(), table_names)); - - let mut threads = Vec::new(); - for _ in 0..app.settings.init_threads_amount { - threads.push(tokio::spawn(super::load_table_files::spawn(app.clone()))); - } - - for thread in threads { - thread.await.unwrap(); - } - - while let Some(db_table) = app.init_state.get_table_data_result().await { - crate::db_operations::write::table::init(app.as_ref(), db_table).await; - } -} diff --git a/src/persist_operations/data_initializer/load_tasks/init_state.rs b/src/persist_operations/data_initializer/load_tasks/init_state.rs deleted file mode 100644 index 4daa5da..0000000 --- a/src/persist_operations/data_initializer/load_tasks/init_state.rs +++ /dev/null @@ -1,112 +0,0 @@ -use std::sync::atomic::{AtomicUsize, Ordering}; - -use crate::{ - persist_io::TableListOfFilesUploader, persist_operations::data_initializer::LoadedTableItem, -}; -use my_no_sql_sdk::core::db::DbTable; -use tokio::sync::Mutex; - -use super::{init_state_data::NextFileToLoadResult, InitStateData, InitStateSnapshot}; - -pub struct InitState { - data: Mutex>, - - tables_total: AtomicUsize, - tables_loaded: AtomicUsize, - - files_total: AtomicUsize, - files_loaded: AtomicUsize, -} - -impl InitState { - pub fn new() -> Self { - Self { - data: Mutex::new(Some(InitStateData::new())), - tables_total: AtomicUsize::new(0), - files_total: AtomicUsize::new(0), - files_loaded: AtomicUsize::new(0), - tables_loaded: AtomicUsize::new(0), - } - } - - fn get_init_state_mut(init_state_data: &mut Option) -> &mut InitStateData { - match init_state_data.as_mut() { - Some(init_state_data) => init_state_data, - None => panic!("Init State Data is disposed"), - } - } - - pub async fn init_table_names(&self, tables: Vec) { - println!("Added tables amount {}", tables.len()); - self.tables_total.store(tables.len(), Ordering::SeqCst); - - let mut write_access = self.data.lock().await; - - Self::get_init_state_mut(&mut write_access).init_table_names(tables); - } - - pub async fn get_next_file_to_load(&self) -> NextFileToLoadResult { - let mut write_access = self.data.lock().await; - Self::get_init_state_mut(&mut write_access).get_next_file_to_load() - } - - pub async fn upload_table_file( - &self, - table_name: &str, - file_name: String, - table_item: LoadedTableItem, - ) { - self.files_loaded.fetch_add(1, Ordering::SeqCst); - let mut write_access = self.data.lock().await; - let write_access = Self::get_init_state_mut(&mut write_access); - if write_access.upload_table_file_content(table_name, file_name, table_item) { - self.tables_loaded.fetch_add(1, Ordering::SeqCst); - } - } - - pub async fn get_snapshot(&self) -> InitStateSnapshot { - InitStateSnapshot { - tables_total: self.tables_total.load(Ordering::SeqCst), - tables_loaded: self.tables_loaded.load(Ordering::SeqCst), - files_total: self.files_total.load(Ordering::SeqCst), - files_loaded: self.files_loaded.load(Ordering::SeqCst), - } - } - - pub async fn get_table_data_result(&self) -> Option { - let mut write_access = self.data.lock().await; - let write_access = Self::get_init_state_mut(&mut write_access); - - let (table_name, task) = write_access.remove_next_task()?; - - let db_table = task.get_result(table_name); - - Some(db_table) - } - - pub async fn dispose(&self) { - let mut write_access = self.data.lock().await; - *write_access = None; - } -} - -#[async_trait::async_trait] -impl TableListOfFilesUploader for InitState { - async fn add_files(&self, table_name: &str, files: Vec) { - self.files_total.fetch_add(files.len(), Ordering::SeqCst); - - let mut write_access = self.data.lock().await; - write_access - .as_mut() - .unwrap() - .add_files_to_table(table_name, files); - } - - async fn set_files_list_is_loaded(&self, table_name: &str) { - let mut write_access = self.data.lock().await; - write_access - .as_mut() - .unwrap() - .set_file_list_is_loaded(table_name) - } -} diff --git a/src/persist_operations/data_initializer/load_tasks/init_state_data.rs b/src/persist_operations/data_initializer/load_tasks/init_state_data.rs deleted file mode 100644 index 26ca380..0000000 --- a/src/persist_operations/data_initializer/load_tasks/init_state_data.rs +++ /dev/null @@ -1,123 +0,0 @@ -use std::collections::BTreeMap; - -use crate::{db_operations::validation, persist_operations::data_initializer::LoadedTableItem}; - -use my_logger::LogEventCtx; - -use super::LoadTableTask; - -pub enum NextFileToLoadResult { - FileToLoad { - table_name: String, - file_name: String, - }, - NotReadyYet, - NothingToLoad, -} - -pub struct InitStateData { - tables: BTreeMap, -} - -impl InitStateData { - pub fn new() -> Self { - Self { - tables: BTreeMap::new(), - } - } - - pub fn init_table_names(&mut self, tables_names: Vec) { - for table_name in tables_names { - if let Err(err) = validation::validate_table_name(table_name.as_str()) { - my_logger::LOGGER.write_error( - "init_tables".to_string(), - format!( - "Table name does not fit validation. Skipping loading it... Reason:{:?}", - err - ), - LogEventCtx::new().add("tableName", table_name), - ); - } else { - self.tables.insert(table_name, LoadTableTask::new()); - } - } - } - - pub fn get_next_file_to_load(&mut self) -> NextFileToLoadResult { - let mut all_files_are_loaded_amount = 0; - - for (table_name, table_task) in &mut self.tables { - if let Some(file_name) = table_task.get_next_file_to_load_content() { - return NextFileToLoadResult::FileToLoad { - table_name: table_name.to_string(), - file_name, - }; - } - - if table_task.is_file_list_loaded() { - all_files_are_loaded_amount += 1; - } - } - - if all_files_are_loaded_amount == self.tables.len() { - return NextFileToLoadResult::NothingToLoad; - } - - return NextFileToLoadResult::NotReadyYet; - } - - pub fn add_files_to_table(&mut self, table_name: &str, files: Vec) { - if let Some(table) = self.tables.get_mut(table_name) { - table.add_list_of_files(files); - } - } - - pub fn set_file_list_is_loaded(&mut self, table_name: &str) { - if let Some(table) = self.tables.get_mut(table_name) { - table.set_file_list_is_loaded(); - } - } - - pub fn upload_table_file_content( - &mut self, - table_name: &str, - file_name: String, - table_item: LoadedTableItem, - ) -> bool { - if let Some(table) = self.tables.get_mut(table_name) { - match table_item { - LoadedTableItem::TableAttributes(mut attrs) => { - if let Some(max_partitions_amount) = attrs.max_partitions_amount { - if max_partitions_amount == 0 { - attrs.max_partitions_amount = None; - } - } - - table.add_attribute(file_name, attrs); - } - LoadedTableItem::DbPartition(db_partition) => { - table.add_db_partition(file_name, db_partition); - } - } - - return table.all_files_are_loaded(); - } - - return false; - } - - fn get_first_table_name(&self) -> Option { - for key in self.tables.keys() { - return Some(key.to_string()); - } - None - } - - pub fn remove_next_task(&mut self) -> Option<(String, LoadTableTask)> { - let table_name = self.get_first_table_name()?; - - let result = self.tables.remove(&table_name)?; - - Some((table_name, result)) - } -} diff --git a/src/persist_operations/data_initializer/load_tasks/init_state_snapshot.rs b/src/persist_operations/data_initializer/load_tasks/init_state_snapshot.rs deleted file mode 100644 index 63341c8..0000000 --- a/src/persist_operations/data_initializer/load_tasks/init_state_snapshot.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub struct InitStateSnapshot { - pub tables_total: usize, - pub tables_loaded: usize, - pub files_total: usize, - pub files_loaded: usize, -} diff --git a/src/persist_operations/data_initializer/load_tasks/load_table_task.rs b/src/persist_operations/data_initializer/load_tasks/load_table_task.rs deleted file mode 100644 index 1794ef0..0000000 --- a/src/persist_operations/data_initializer/load_tasks/load_table_task.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::collections::BTreeMap; - -use my_no_sql_sdk::core::db::{DbPartition, DbTable, DbTableAttributes}; - -pub enum FileStatus { - Waiting, - Loading, - DbPartition(DbPartition), -} - -impl FileStatus { - pub fn is_waiting(&self) -> bool { - match self { - FileStatus::Waiting => true, - _ => false, - } - } -} - -pub struct LoadTableTask { - files_list_is_loaded: bool, - files: BTreeMap, - attr: Option, -} - -impl LoadTableTask { - pub fn new() -> Self { - Self { - files_list_is_loaded: false, - files: BTreeMap::new(), - attr: None, - } - } - - pub fn add_list_of_files(&mut self, files: Vec) { - for file in files { - self.files.insert(file, FileStatus::Waiting); - } - } - - pub fn get_next_file_to_load_content(&mut self) -> Option { - let next_file = self.get_next_file_name_to_load_content()?; - - self.files - .insert(next_file.to_string(), FileStatus::Loading); - - Some(next_file) - } - - fn get_next_file_name_to_load_content(&self) -> Option { - for (file_name, status) in &self.files { - if status.is_waiting() { - return Some(file_name.clone()); - } - } - - None - } - - pub fn add_db_partition(&mut self, file_name: String, db_partition: DbPartition) { - self.files - .insert(file_name, FileStatus::DbPartition(db_partition)); - } - - pub fn add_attribute(&mut self, file_name: String, attr: DbTableAttributes) { - self.attr = Some(attr); - self.files.remove(file_name.as_str()); - } - - pub fn is_file_list_loaded(&self) -> bool { - self.files_list_is_loaded - } - - pub fn set_file_list_is_loaded(&mut self) { - self.files_list_is_loaded = true; - } - - pub fn get_result(mut self, table_name: String) -> DbTable { - let mut attr = self.attr.take(); - - if attr.is_none() { - attr = Some(DbTableAttributes::create_default()) - } - - let mut db_table = DbTable::new(table_name, attr.unwrap()); - - for (_, file_status) in self.files { - match file_status { - FileStatus::Waiting => { - panic!("Somehow we started getting result having Waiting File") - } - FileStatus::Loading => { - panic!("Somehow we started getting result having Loading File") - } - FileStatus::DbPartition(db_partition) => { - for db_row in db_partition.get_all_rows() { - db_table.avg_size.add(db_row); - } - db_table.partitions.insert(db_partition); - } - } - } - - db_table - } - - pub fn all_files_are_loaded(&self) -> bool { - for file in self.files.values() { - match file { - FileStatus::Waiting => return false, - FileStatus::Loading => { - return false; - } - FileStatus::DbPartition(_) => {} - } - } - - true - } -} diff --git a/src/persist_operations/data_initializer/load_tasks/mod.rs b/src/persist_operations/data_initializer/load_tasks/mod.rs deleted file mode 100644 index 313864e..0000000 --- a/src/persist_operations/data_initializer/load_tasks/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod init_state; -mod init_state_data; -mod init_state_snapshot; -mod load_table_task; - -pub use init_state::InitState; -pub use init_state_data::{InitStateData, NextFileToLoadResult}; -pub use init_state_snapshot::InitStateSnapshot; -pub use load_table_task::LoadTableTask; diff --git a/src/persist_operations/data_initializer/loaded_table_item.rs b/src/persist_operations/data_initializer/loaded_table_item.rs deleted file mode 100644 index 297e2fe..0000000 --- a/src/persist_operations/data_initializer/loaded_table_item.rs +++ /dev/null @@ -1,28 +0,0 @@ -use my_no_sql_sdk::core::db::{DbPartition, DbTableAttributes}; - -use crate::{persist_io::TableFile, persist_operations::serializers::TableMetadataFileContract}; - -pub enum LoadedTableItem { - TableAttributes(DbTableAttributes), - DbPartition(DbPartition), -} - -impl LoadedTableItem { - pub fn new(table_file: &TableFile, content: &[u8]) -> Result { - match table_file { - TableFile::TableAttributes => { - let table_metadata = TableMetadataFileContract::parse(content); - let result = LoadedTableItem::TableAttributes(table_metadata.into()); - return Ok(result); - } - TableFile::DbPartition(partition_key) => { - let db_partition = - crate::persist_operations::serializers::db_partition::deserialize( - partition_key.as_str(), - content, - )?; - return Ok(LoadedTableItem::DbPartition(db_partition)); - } - } - } -} diff --git a/src/persist_operations/data_initializer/mod.rs b/src/persist_operations/data_initializer/mod.rs deleted file mode 100644 index 9b31b61..0000000 --- a/src/persist_operations/data_initializer/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -mod load_table_files; -mod load_tables; -pub mod load_tasks; -mod loaded_table_item; -mod table_list_of_files_loader; - -pub use load_tables::load_tables; -pub use loaded_table_item::LoadedTableItem; -pub use table_list_of_files_loader::table_list_of_files_loader; -pub mod from_other_instance; diff --git a/src/persist_operations/data_initializer/table_list_of_files_loader.rs b/src/persist_operations/data_initializer/table_list_of_files_loader.rs deleted file mode 100644 index 70d40eb..0000000 --- a/src/persist_operations/data_initializer/table_list_of_files_loader.rs +++ /dev/null @@ -1,11 +0,0 @@ -use std::sync::Arc; - -use crate::app::AppContext; - -pub async fn table_list_of_files_loader(app: Arc, table_names: Vec) { - for table_name in table_names { - app.persist_io - .get_table_files(&table_name, &app.init_state) - .await; - } -} diff --git a/src/persist_operations/mod.rs b/src/persist_operations/mod.rs deleted file mode 100644 index 3d93782..0000000 --- a/src/persist_operations/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod blob_content_cache; -pub mod data_initializer; -pub mod serializers; -pub mod sync; diff --git a/src/persist_operations/serializers/mod.rs b/src/persist_operations/serializers/mod.rs deleted file mode 100644 index a8bbc4d..0000000 --- a/src/persist_operations/serializers/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod db_partition; -pub mod table_attrs; -pub use table_attrs::TableMetadataFileContract; diff --git a/src/persist_operations/sync/create_table.rs b/src/persist_operations/sync/create_table.rs deleted file mode 100644 index 8ca873c..0000000 --- a/src/persist_operations/sync/create_table.rs +++ /dev/null @@ -1,17 +0,0 @@ -use my_no_sql_server_core::DbTableWrapper; - -use crate::app::AppContext; - -pub async fn create_table(app: &AppContext, db_table: &DbTableWrapper) { - app.persist_io - .create_table_folder(db_table.name.as_str()) - .await; - - let attrs = db_table.get_attributes().await; - - app.blob_content_cache - .create_table(db_table.name.as_str(), &attrs) - .await; - - super::save_table_attributes(app, db_table.name.as_str(), &attrs).await; -} diff --git a/src/persist_operations/sync/delete_partition.rs b/src/persist_operations/sync/delete_partition.rs deleted file mode 100644 index b1c1c11..0000000 --- a/src/persist_operations/sync/delete_partition.rs +++ /dev/null @@ -1,27 +0,0 @@ -use my_no_sql_sdk::core::db::PartitionKey; - -use crate::{ - app::AppContext, persist_io::TableFile, - persist_operations::blob_content_cache::BlobPartitionUpdateTimeResult, -}; - -pub async fn delete_partition(app: &AppContext, table_name: &str, partition_key: PartitionKey) { - let partition_in_blob = app - .blob_content_cache - .get(table_name, partition_key.as_str()) - .await; - - match partition_in_blob { - BlobPartitionUpdateTimeResult::Ok(_) => { - app.persist_io - .delete_table_file(table_name, &TableFile::DbPartition(partition_key.clone())) - .await; - - app.blob_content_cache - .delete_table_partition(table_name, partition_key.as_str()) - .await; - } - BlobPartitionUpdateTimeResult::TableNotFound => {} - BlobPartitionUpdateTimeResult::PartitionNoFound => {} - } -} diff --git a/src/persist_operations/sync/delete_table.rs b/src/persist_operations/sync/delete_table.rs deleted file mode 100644 index 34cf6b1..0000000 --- a/src/persist_operations/sync/delete_table.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::sync::Arc; - -use crate::app::AppContext; - -pub async fn delete_table(app: Arc, table_name: String) { - if !app.blob_content_cache.has_table(table_name.as_str()).await { - return; - } - - app.persist_io - .delete_table_folder(table_name.as_str()) - .await; - - app.blob_content_cache - .delete_table(table_name.as_str()) - .await; -} diff --git a/src/persist_operations/sync/mod.rs b/src/persist_operations/sync/mod.rs deleted file mode 100644 index d33b73b..0000000 --- a/src/persist_operations/sync/mod.rs +++ /dev/null @@ -1,14 +0,0 @@ -mod create_table; -mod delete_partition; -mod delete_table; -mod save_partition; -mod save_table; -mod save_table_attributes; -mod upload_partition; -pub use create_table::create_table; -pub use delete_partition::delete_partition; -pub use delete_table::delete_table; -pub use save_partition::save_partition; -pub use save_table::save_table; -pub use save_table_attributes::save_table_attributes; -use upload_partition::upload_partition; diff --git a/src/persist_operations/sync/save_partition.rs b/src/persist_operations/sync/save_partition.rs deleted file mode 100644 index a316e54..0000000 --- a/src/persist_operations/sync/save_partition.rs +++ /dev/null @@ -1,51 +0,0 @@ -use my_no_sql_sdk::core::db::PartitionKey; -use my_no_sql_server_core::DbTableWrapper; - -use crate::{ - app::AppContext, persist_operations::blob_content_cache::BlobPartitionUpdateTimeResult, -}; - -use super::super::sync; - -pub async fn save_partition( - app: &AppContext, - db_table: &DbTableWrapper, - partition_key: PartitionKey, -) { - let get_blob_content_cache = app - .blob_content_cache - .get(db_table.name.as_str(), partition_key.as_str()) - .await; - - let partition_snapshot = db_table - .get_partition_snapshot(partition_key.as_str()) - .await; - - match get_blob_content_cache { - BlobPartitionUpdateTimeResult::Ok(blob_date_time) => { - if partition_snapshot.is_none() { - sync::delete_partition(app, db_table.name.as_str(), partition_key).await; - return; - } - - let partition_snapshot = partition_snapshot.unwrap(); - - if partition_snapshot.last_write_moment.unix_microseconds - > blob_date_time.unix_microseconds - { - sync::upload_partition(app, db_table.name.as_str(), partition_snapshot).await; - } - } - BlobPartitionUpdateTimeResult::TableNotFound => { - if let Some(snapshot) = partition_snapshot { - sync::create_table(app, db_table).await; - sync::upload_partition(app, db_table.name.as_str(), snapshot).await; - } - } - BlobPartitionUpdateTimeResult::PartitionNoFound => { - if let Some(snapshot) = partition_snapshot { - sync::upload_partition(app, db_table.name.as_str(), snapshot).await; - } - } - } -} diff --git a/src/persist_operations/sync/save_table.rs b/src/persist_operations/sync/save_table.rs deleted file mode 100644 index fc5ace6..0000000 --- a/src/persist_operations/sync/save_table.rs +++ /dev/null @@ -1,55 +0,0 @@ -use my_no_sql_sdk::core::db::db_table_master_node::PartitionLastWriteMoment; -use my_no_sql_sdk::core::rust_extensions::sorted_vec::SortedVecWithStrKey; -use my_no_sql_server_core::{db_snapshots::DbTableSnapshot, DbTableWrapper}; - -use crate::app::AppContext; - -use super::super::sync; - -pub async fn save_table(app: &AppContext, db_table: &DbTableWrapper) { - let snapshot: DbTableSnapshot = db_table.get_table_snapshot().await; - - let in_blob = app - .blob_content_cache - .get_snapshot(db_table.name.as_str()) - .await; - - match in_blob { - Some(in_blob) => { - sync_with_blob(app, db_table.name.as_str(), in_blob, snapshot).await; - } - None => { - init_new_table(app, db_table.name.as_str(), snapshot).await; - } - } -} - -async fn init_new_table(app: &AppContext, table_name: &str, snapshot: DbTableSnapshot) { - for snapshot in snapshot.by_partition { - sync::upload_partition(app, table_name, snapshot).await; - } -} - -async fn sync_with_blob( - app: &AppContext, - table_name: &str, - mut in_blob: SortedVecWithStrKey, - snapshot: DbTableSnapshot, -) { - for partition_snapshot in snapshot.by_partition { - match in_blob.remove(partition_snapshot.partition_key.as_str()) { - Some(snapshot_in_blob) => { - if partition_snapshot.has_to_persist(snapshot_in_blob.last_write_moment) { - sync::upload_partition(app, table_name, partition_snapshot).await; - } - } - None => { - sync::upload_partition(app, table_name, partition_snapshot).await; - } - } - } - - for item in in_blob.into_vec() { - sync::delete_partition(app, table_name, item.partition_key).await; - } -} diff --git a/src/persist_operations/sync/save_table_attributes.rs b/src/persist_operations/sync/save_table_attributes.rs deleted file mode 100644 index 912d4a5..0000000 --- a/src/persist_operations/sync/save_table_attributes.rs +++ /dev/null @@ -1,17 +0,0 @@ -use my_no_sql_sdk::core::db::DbTableAttributes; - -use super::super::serializers; -use crate::{app::AppContext, persist_io::TableFile}; -pub async fn save_table_attributes(app: &AppContext, table_name: &str, attrs: &DbTableAttributes) { - app.persist_io.create_table_folder(table_name).await; - - let content = serializers::table_attrs::serialize(attrs); - - app.persist_io - .save_table_file(table_name, &TableFile::TableAttributes, content) - .await; - - app.blob_content_cache - .update_table_attributes(table_name, attrs.clone()) - .await; -} diff --git a/src/persist_operations/sync/upload_partition.rs b/src/persist_operations/sync/upload_partition.rs deleted file mode 100644 index 396353a..0000000 --- a/src/persist_operations/sync/upload_partition.rs +++ /dev/null @@ -1,19 +0,0 @@ -use my_no_sql_server_core::db_snapshots::DbPartitionSnapshot; - -use crate::{app::AppContext, persist_io::TableFile}; - -pub async fn upload_partition(app: &AppContext, table_name: &str, snapshot: DbPartitionSnapshot) { - let content = snapshot.db_rows_snapshot.as_json_array(); - - app.persist_io - .save_table_file( - table_name, - &TableFile::DbPartition(snapshot.partition_key.clone()), - content.build(), - ) - .await; - - app.blob_content_cache - .update_table_partition_snapshot_id(table_name, snapshot) - .await; -} diff --git a/src/scripts/mod.rs b/src/scripts/mod.rs new file mode 100644 index 0000000..fd0da9b --- /dev/null +++ b/src/scripts/mod.rs @@ -0,0 +1,2 @@ +pub mod serializers; +pub const TABLE_METADATA_FILE_NAME: &str = ".metadata"; diff --git a/src/persist_operations/serializers/db_partition.rs b/src/scripts/serializers/db_partition.rs similarity index 95% rename from src/persist_operations/serializers/db_partition.rs rename to src/scripts/serializers/db_partition.rs index 70bc39b..9dae2c6 100644 --- a/src/persist_operations/serializers/db_partition.rs +++ b/src/scripts/serializers/db_partition.rs @@ -24,7 +24,7 @@ pub fn deserialize(partition_key: &str, raw: &[u8]) -> Result { if db_row.get_partition_key() == partition_key { db_partition.insert_row(Arc::new(db_row)); diff --git a/src/scripts/serializers/mod.rs b/src/scripts/serializers/mod.rs new file mode 100644 index 0000000..af5e52e --- /dev/null +++ b/src/scripts/serializers/mod.rs @@ -0,0 +1,2 @@ +pub mod db_partition; +pub mod table_attrs; diff --git a/src/persist_operations/serializers/table_attrs.rs b/src/scripts/serializers/table_attrs.rs similarity index 100% rename from src/persist_operations/serializers/table_attrs.rs rename to src/scripts/serializers/table_attrs.rs diff --git a/src/settings_reader.rs b/src/settings_reader.rs index a849fa3..f5c94fa 100644 --- a/src/settings_reader.rs +++ b/src/settings_reader.rs @@ -1,10 +1,8 @@ -use my_azure_storage_sdk::AzureStorageConnection; use my_no_sql_sdk::core::rust_extensions::StrOrString; +use my_no_sql_server_core::rust_extensions; use serde::{Deserialize, Serialize}; -use std::{env, sync::Arc}; -use tokio::{fs::File, io::AsyncReadExt}; -use crate::{persist_io::PersistIoOperations, sqlite_repo::SqlLiteRepo}; +use crate::sqlite_repo::SqlLiteRepo; #[derive(Serialize, Deserialize, Debug)] pub struct SettingsModel { @@ -46,16 +44,11 @@ pub struct SettingsModel { } impl SettingsModel { - pub async fn get_persist_io(&self) -> PersistIoOperations { - if self.persistence_dest.as_str().ends_with(".sqlite") { - let file = my_no_sql_server_core::rust_extensions::file_utils::format_path( - self.persistence_dest.as_str(), - ); - let sqlite_repo = SqlLiteRepo::new(file.to_string()).await; - return PersistIoOperations::as_sqlite(sqlite_repo); - } - let conn_string = AzureStorageConnection::from_conn_string(self.persistence_dest.as_str()); - PersistIoOperations::as_azure_connection(Arc::new(conn_string)) + pub async fn get_sqlite_repo(&self) -> SqlLiteRepo { + let file = my_no_sql_server_core::rust_extensions::file_utils::format_path( + self.persistence_dest.as_str(), + ); + SqlLiteRepo::new(file.to_string()).await } pub fn get_backup_folder<'s>(&'s self) -> StrOrString<'s> { @@ -72,18 +65,26 @@ impl SettingsModel { } pub async fn read_settings() -> SettingsModel { - let file_name = get_settings_filename(); + let file_name = rust_extensions::file_utils::format_path("~/.mynosqlserver"); + + let file_content = tokio::fs::read(file_name.as_str()).await; - let mut file = File::open(file_name).await.unwrap(); + if let Err(err) = &file_content { + panic!( + "Can't open settings file [{}]. Err: {}", + file_name.as_str(), + err + ); + } - let mut file_content: Vec = vec![]; - file.read_to_end(&mut file_content).await.unwrap(); + let file_content = file_content.unwrap(); let result: SettingsModel = serde_yaml::from_slice(file_content.as_slice()).unwrap(); result } +/* fn get_settings_filename() -> String { let path = env!("HOME"); @@ -93,3 +94,4 @@ fn get_settings_filename() -> String { return format!("{}{}", path, "/.mynosqlserver"); } + */ diff --git a/src/sqlite_repo/entity_dto.rs b/src/sqlite_repo/entity_dto.rs new file mode 100644 index 0000000..5903f39 --- /dev/null +++ b/src/sqlite_repo/entity_dto.rs @@ -0,0 +1,45 @@ +use my_no_sql_sdk::core::db::DbRow; +use my_sqlite::macros::*; + +#[derive(TableSchema, InsertDbEntity, UpdateDbEntity, SelectDbEntity, Debug)] +pub struct MyNoSqlEntityDto { + #[primary_key(0)] + pub table_name: String, + #[primary_key(1)] + pub partition_key: String, + #[primary_key(2)] + pub row_key: String, + pub content: String, +} + +impl MyNoSqlEntityDto { + pub fn from_db_row(table_name: &str, db_row: &DbRow) -> Self { + Self { + table_name: table_name.to_string(), + partition_key: db_row.get_partition_key().to_string(), + row_key: db_row.get_row_key().to_string(), + + content: std::str::from_utf8(db_row.get_src_as_slice()) + .unwrap() + .to_string(), + } + } +} + +#[derive(WhereDbModel, Debug)] +pub struct WhereEntityByTableName<'s> { + pub table_name: &'s str, +} + +#[derive(WhereDbModel, Debug)] +pub struct WhereEntityByPartitionKey<'s> { + pub table_name: &'s str, + pub partition_key: &'s str, +} + +#[derive(WhereDbModel, Debug)] +pub struct DeleteMyNoSqlEntityWhereModel<'s> { + pub table_name: &'s str, + pub partition_key: &'s str, + pub row_key: &'s str, +} diff --git a/src/sqlite_repo/row_dto.rs b/src/sqlite_repo/file_dto.rs similarity index 100% rename from src/sqlite_repo/row_dto.rs rename to src/sqlite_repo/file_dto.rs diff --git a/src/sqlite_repo/mod.rs b/src/sqlite_repo/mod.rs index f48d770..d036dc6 100644 --- a/src/sqlite_repo/mod.rs +++ b/src/sqlite_repo/mod.rs @@ -1,6 +1,10 @@ mod sqlite_repo; pub use sqlite_repo::*; -mod row_dto; -pub use row_dto::*; +//mod file_dto; +//pub use file_dto::*; //mod table_dto; //pub use table_dto::*; +mod entity_dto; +pub use entity_dto::*; +mod table_metadata_dto; +pub use table_metadata_dto::*; diff --git a/src/sqlite_repo/sqlite_repo.rs b/src/sqlite_repo/sqlite_repo.rs index 6deca34..4cf096f 100644 --- a/src/sqlite_repo/sqlite_repo.rs +++ b/src/sqlite_repo/sqlite_repo.rs @@ -1,9 +1,15 @@ +use my_no_sql_sdk::core::db::{DbTableAttributes, DbTableName, PartitionKey}; use my_sqlite::sql_where::NoneWhereModel; use my_sqlite::{SqlLiteConnection, SqlLiteConnectionBuilder}; -use super::row_dto::*; +use super::{ + DeleteMyNoSqlEntityWhereModel, DeleteTableMetadataWhereModel, MyNoSqlEntityDto, + TableMetaDataDto, WhereEntityByPartitionKey, WhereEntityByTableName, +}; -pub const FILES_TABLE: &str = "files"; +//pub const FILES_TABLE: &str = "files"; +pub const ENTITIES_TABLE: &str = "entities"; +pub const TABLES_METADATA_TABLE: &str = "tables_metadata"; pub struct SqlLiteRepo { sqlite: SqlLiteConnection, @@ -13,13 +19,99 @@ impl SqlLiteRepo { pub async fn new(file_name: String) -> Self { Self { sqlite: SqlLiteConnectionBuilder::new(file_name) - .create_table_if_no_exists::(FILES_TABLE) + // .create_table_if_no_exists::(FILES_TABLE) + .create_table_if_no_exists::(ENTITIES_TABLE) + .create_table_if_no_exists::(TABLES_METADATA_TABLE) .build() .await .unwrap(), } } + pub async fn save_entities(&self, entities: &[MyNoSqlEntityDto]) { + self.sqlite + .bulk_insert_or_update(entities, ENTITIES_TABLE) + .await + .unwrap(); + } + + pub async fn get_all_entities(&self) -> Vec { + self.sqlite + .query_rows(ENTITIES_TABLE, NoneWhereModel::new()) + .await + .unwrap() + } + + pub async fn delete_entity( + &self, + table_name: &DbTableName, + partition_key: &str, + row_key: &str, + ) { + let where_model = DeleteMyNoSqlEntityWhereModel { + table_name: table_name.as_str(), + partition_key: partition_key, + row_key, + }; + self.sqlite + .delete_db_entity(ENTITIES_TABLE, &where_model) + .await + .unwrap(); + } + + pub async fn clean_table_content(&self, table_name: &DbTableName) { + let where_model = WhereEntityByTableName { + table_name: table_name.as_str(), + }; + self.sqlite + .delete_db_entity(ENTITIES_TABLE, &where_model) + .await + .unwrap(); + } + + pub async fn clean_partition_content( + &self, + table_name: &DbTableName, + partition_key: &PartitionKey, + ) { + let where_model = WhereEntityByPartitionKey { + table_name: table_name.as_str(), + partition_key: partition_key.as_str(), + }; + self.sqlite + .delete_db_entity(ENTITIES_TABLE, &where_model) + .await + .unwrap(); + } + + pub async fn save_table_metadata(&self, table_name: &DbTableName, attr: &DbTableAttributes) { + let dto = TableMetaDataDto::from_table_attr(table_name.as_str(), &attr); + + self.sqlite + .insert_or_update_db_entity(TABLES_METADATA_TABLE, &dto) + .await + .unwrap(); + } + + pub async fn delete_table_metadata(&self, table_name: &DbTableName) { + let dto = DeleteTableMetadataWhereModel { + table_name: table_name.as_str(), + }; + + self.sqlite + .delete_db_entity(TABLES_METADATA_TABLE, &dto) + .await + .unwrap(); + } + + pub async fn get_tables(&self) -> Vec { + self.sqlite + .query_rows(TABLES_METADATA_TABLE, NoneWhereModel::new()) + .await + .unwrap() + } + + /* pub async fn save_file(&self, table_name: &str, file_name: &str, content: String) { let table = MyNoSqlFileDto { table_name: table_name.to_string(), @@ -57,4 +149,5 @@ impl SqlLiteRepo { .await .unwrap(); } + */ } diff --git a/src/sqlite_repo/table_dto.rs b/src/sqlite_repo/table_dto.rs deleted file mode 100644 index ca649cf..0000000 --- a/src/sqlite_repo/table_dto.rs +++ /dev/null @@ -1,28 +0,0 @@ -use my_sqlite::macros::*; - -use crate::persist_operations::serializers::TableMetadataFileContract; - -#[derive(TableSchema, InsertDbEntity, UpdateDbEntity, SelectDbEntity, Debug)] -pub struct TableAttributesDto { - #[primary_key(0)] - pub table_name: String, - pub created_at: String, - pub persist: bool, - pub max_partitions_amount: Option, - pub max_rows_per_partition_amount: Option, -} -#[derive(Debug, WhereDbModel)] -pub struct TableAttributesWhereModel<'s> { - pub table_name: &'s str, -} - -impl Into for TableAttributesDto { - fn into(self) -> TableMetadataFileContract { - TableMetadataFileContract { - persist: self.persist, - max_partitions_amount: self.max_partitions_amount.map(|x| x as usize), - max_rows_per_partition_amount: self.max_rows_per_partition_amount.map(|x| x as usize), - created: self.created_at.into(), - } - } -} diff --git a/src/sqlite_repo/table_metadata_dto.rs b/src/sqlite_repo/table_metadata_dto.rs new file mode 100644 index 0000000..0ba1d80 --- /dev/null +++ b/src/sqlite_repo/table_metadata_dto.rs @@ -0,0 +1,52 @@ +use my_no_sql_sdk::core::db::{DbTableAttributes, DbTableName}; +use my_no_sql_server_core::rust_extensions::date_time::DateTimeAsMicroseconds; +use my_sqlite::macros::*; + +use crate::operations::init::TableAttributeInitContract; +#[derive(TableSchema, InsertDbEntity, UpdateDbEntity, SelectDbEntity, Debug)] +pub struct TableMetaDataDto { + #[primary_key(0)] + pub table_name: String, + pub max_partitions_amount: i64, + pub max_rows_per_partition_amount: i64, + pub persist: bool, + pub created: i64, +} + +impl TableMetaDataDto { + pub fn from_table_attr(table_name: &str, attr: &DbTableAttributes) -> Self { + Self { + table_name: table_name.to_string(), + max_partitions_amount: attr.max_partitions_amount.unwrap_or(0) as i64, + max_rows_per_partition_amount: attr.max_rows_per_partition_amount.unwrap_or(0) as i64, + persist: attr.persist, + created: attr.created.unix_microseconds, + } + } +} + +impl TableAttributeInitContract for TableMetaDataDto { + fn into(self) -> (DbTableName, DbTableAttributes) { + let attr = DbTableAttributes { + max_partitions_amount: if self.max_partitions_amount <= 0 { + None + } else { + Some(self.max_partitions_amount as usize) + }, + max_rows_per_partition_amount: if self.max_rows_per_partition_amount <= 0 { + None + } else { + Some(self.max_rows_per_partition_amount as usize) + }, + persist: self.persist, + created: DateTimeAsMicroseconds::new(self.created), + }; + + (self.table_name.into(), attr) + } +} + +#[derive(Debug, WhereDbModel)] +pub struct DeleteTableMetadataWhereModel<'s> { + pub table_name: &'s str, +} diff --git a/src/tcp/tcp_server_events.rs b/src/tcp/tcp_server_events.rs index 0cb49e9..47dcd12 100644 --- a/src/tcp/tcp_server_events.rs +++ b/src/tcp/tcp_server_events.rs @@ -234,9 +234,8 @@ impl SocketEventCallback for if let Some(db_table) = &db_table { for (partition_key, set_expiration_time) in partitions { crate::db_operations::update_partition_expiration_time( - &self.app, db_table, - &partition_key, + partition_key, set_expiration_time, ) } diff --git a/src/zip/db_zip_builder.rs b/src/zip/db_zip_builder.rs index 339a60b..ffb2e37 100644 --- a/src/zip/db_zip_builder.rs +++ b/src/zip/db_zip_builder.rs @@ -25,14 +25,14 @@ impl DbZipBuilder { let file_name = format!( "{}/{}", table_name, - crate::persist_io::TABLE_METADATA_FILE_NAME + crate::scripts::TABLE_METADATA_FILE_NAME ); let options = zip::write::SimpleFileOptions::default() .compression_method(zip::CompressionMethod::Deflated); self.zip_writer.start_file(file_name, options)?; - let payload = crate::persist_operations::serializers::table_attrs::serialize(&content.attr); + let payload = crate::scripts::serializers::table_attrs::serialize(&content.attr); write_to_zip_file(&mut self.zip_writer, &payload)?; for itm in &content.by_partition { diff --git a/test/test-case.json b/test/test-case.json deleted file mode 100644 index 4ec10f8..0000000 --- a/test/test-case.json +++ /dev/null @@ -1 +0,0 @@ -[{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"ALGOBTC","CurrentPrice":0.00002302,"H24P":0.05,"H24":{"Price":0.00002301,"RecordTime":"2021-08-27T10:19:22.520141Z"},"D7":{"Price":0.00002301,"RecordTime":"2021-08-27T10:27:33.897397Z"},"M1":{"Price":0.00002301,"RecordTime":"2021-08-27T10:27:36.166979Z"},"M3":{"Price":0.00002301,"RecordTime":"2021-08-27T10:27:38.86826Z"}},"PartitionKey":"jetwallet","RowKey":"ALGOBTC","TimeStamp":"2021-08-27T10:27:39.0227230Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"ALGOUSD","CurrentPrice":1.1578,"H24P":0.03,"H24":{"Price":1.1575,"RecordTime":"2021-08-27T10:19:28.13551Z"},"D7":{"Price":1.1575,"RecordTime":"2021-08-27T10:27:40.587078Z"},"M1":{"Price":0.8498,"RecordTime":"2021-08-27T10:27:42.794889Z"},"M3":{"Price":0.8854,"RecordTime":"2021-08-27T10:27:45.689835Z"}},"PartitionKey":"jetwallet","RowKey":"ALGOUSD","TimeStamp":"2021-08-27T10:27:45.8456728Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"BCHBTC","CurrentPrice":0.013668,"H24P":0.01,"H24":{"Price":0.013667,"RecordTime":"2021-08-27T10:19:31.277918Z"},"D7":{"Price":0.013667,"RecordTime":"2021-08-27T10:27:48.111119Z"},"M1":{"Price":0.01279,"RecordTime":"2021-08-27T10:27:49.637483Z"},"M3":{"Price":0.01506,"RecordTime":"2021-08-27T10:27:51.153701Z"}},"PartitionKey":"jetwallet","RowKey":"BCHBTC","TimeStamp":"2021-08-27T10:27:51.3336778Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"BCHUSD","CurrentPrice":687.68,"H24P":0.03,"H24":{"Price":687.52,"RecordTime":"2021-08-27T10:19:33.084996Z"},"D7":{"Price":687.52,"RecordTime":"2021-08-27T10:27:53.077706Z"},"M1":{"Price":511.97,"RecordTime":"2021-08-27T10:27:55.707874Z"},"M3":{"Price":527.78,"RecordTime":"2021-08-27T10:28:00.373108Z"}},"PartitionKey":"jetwallet","RowKey":"BCHUSD","TimeStamp":"2021-08-27T10:28:00.5273173Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"BTCEUR","CurrentPrice":43084.94,"H24P":0.03,"H24":{"Price":43074.29,"RecordTime":"2021-08-27T10:19:34.540693Z"},"D7":{"Price":43074.29,"RecordTime":"2021-08-27T10:28:03.463373Z"},"M1":{"Price":33991.97,"RecordTime":"2021-08-27T10:28:05.852796Z"},"M3":{"Price":29641.53,"RecordTime":"2021-08-27T10:28:08.64383Z"}},"PartitionKey":"jetwallet","RowKey":"BTCEUR","TimeStamp":"2021-08-27T10:28:08.8227071Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"BTCUSD","CurrentPrice":50301.96,"H24P":0.03,"H24":{"Price":50291.0,"RecordTime":"2021-08-27T10:20:20.675712Z"},"D7":{"Price":50291.0,"RecordTime":"2021-08-27T10:30:07.592626Z"},"M1":{"Price":40077.04,"RecordTime":"2021-08-27T10:31:16.782437Z"},"M3":{"Price":35092.06,"RecordTime":"2021-08-27T10:34:02.709802Z"}},"PartitionKey":"jetwallet","RowKey":"BTCUSD","TimeStamp":"2021-08-27T10:34:02.9930177Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"DASHBTC","CurrentPrice":0.005537,"H24P":0.33,"H24":{"Price":0.005519,"RecordTime":"2021-08-27T10:20:22.859153Z"},"D7":{"Price":0.005519,"RecordTime":"2021-08-27T10:34:36.600074Z"},"M1":{"Price":0.005519,"RecordTime":"2021-08-27T10:34:43.501571Z"},"M3":{"Price":0.005519,"RecordTime":"2021-08-27T10:34:43.602666Z"}},"PartitionKey":"jetwallet","RowKey":"DASHBTC","TimeStamp":"2021-08-27T10:34:43.7561828Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"DASHUSD","CurrentPrice":278.51,"H24P":0.36,"H24":{"Price":277.52,"RecordTime":"2021-08-27T10:21:09.486934Z"},"D7":{"Price":277.52,"RecordTime":"2021-08-27T10:34:43.690196Z"},"M1":{"Price":152.87,"RecordTime":"2021-08-27T10:34:43.785673Z"},"M3":{"Price":145.74,"RecordTime":"2021-08-27T10:34:43.834666Z"}},"PartitionKey":"jetwallet","RowKey":"DASHUSD","TimeStamp":"2021-08-27T10:34:43.9841451Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"EOSBTC","CurrentPrice":0.00011145,"H24P":0.0,"H24":{"Price":0.00011145,"RecordTime":"2021-08-27T10:21:45.601017Z"},"D7":{"Price":0.00011145,"RecordTime":"2021-08-27T10:34:43.885854Z"},"M1":{"Price":0.00011145,"RecordTime":"2021-08-27T10:34:43.936371Z"},"M3":{"Price":0.00011145,"RecordTime":"2021-08-27T10:34:43.986313Z"}},"PartitionKey":"jetwallet","RowKey":"EOSBTC","TimeStamp":"2021-08-27T10:34:44.1323687Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"EOSUSD","CurrentPrice":5.61,"H24P":0.18,"H24":{"Price":5.6,"RecordTime":"2021-08-27T10:21:50.490412Z"},"D7":{"Price":5.6,"RecordTime":"2021-08-27T10:34:44.03271Z"},"M1":{"Price":3.93,"RecordTime":"2021-08-27T10:34:44.079744Z"},"M3":{"Price":4.15,"RecordTime":"2021-08-27T10:34:44.132191Z"}},"PartitionKey":"jetwallet","RowKey":"EOSUSD","TimeStamp":"2021-08-27T10:34:44.2776344Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"ETHBTC","CurrentPrice":0.06644,"H24P":0.00,"H24":{"Price":0.0664425,"RecordTime":"2021-08-27T10:21:54.945549Z"},"D7":{"Price":0.0664425,"RecordTime":"2021-08-27T10:34:44.177756Z"},"M1":{"Price":0.06740125,"RecordTime":"2021-08-27T10:34:44.228406Z"},"M3":{"Price":0.05916,"RecordTime":"2021-08-27T10:34:44.280904Z"}},"PartitionKey":"jetwallet","RowKey":"ETHBTC","TimeStamp":"2021-08-27T10:34:44.4341130Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"ETHEUR","CurrentPrice":2851.25,"H24P":0.02,"H24":{"Price":2850.9,"RecordTime":"2021-08-27T10:21:58.573395Z"},"D7":{"Price":2850.9,"RecordTime":"2021-08-27T10:34:44.335758Z"},"M1":{"Price":1954.11,"RecordTime":"2021-08-27T10:34:44.387248Z"},"M3":{"Price":1923.27,"RecordTime":"2021-08-27T10:34:44.439264Z"}},"PartitionKey":"jetwallet","RowKey":"ETHEUR","TimeStamp":"2021-08-27T10:34:44.5883154Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"ETHUSD","CurrentPrice":3036.45,"H24P":0.06,"H24":{"Price":3034.75,"RecordTime":"2021-08-20T11:28:37.5326308Z"},"D7":{"Price":3342.25,"RecordTime":"2021-08-27T10:34:44.488701Z"},"M1":{"Price":2305.01,"RecordTime":"2021-08-27T10:34:44.534825Z"},"M3":{"Price":2279.09,"RecordTime":"2021-08-27T10:34:44.605971Z"}},"PartitionKey":"jetwallet","RowKey":"ETHUSD","TimeStamp":"2021-08-27T10:34:44.7627134Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"EURUSD","CurrentPrice":0.0,"H24P":0.0,"H24":{"Price":0.0,"RecordTime":"0001-01-01T00:00:00"},"D7":{"Price":1.17,"RecordTime":"2021-08-27T10:34:44.663006Z"},"M1":{"Price":1.17,"RecordTime":"2021-08-27T10:34:44.707026Z"},"M3":{"Price":1.17,"RecordTime":"2021-08-27T10:34:44.752783Z"}},"PartitionKey":"jetwallet","RowKey":"EURUSD","TimeStamp":"2021-08-27T10:34:44.9043514Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"LTCBTC","CurrentPrice":0.0037545,"H24P":0.04,"H24":{"Price":0.003753,"RecordTime":"2021-08-20T11:28:37.5570669Z"},"D7":{"Price":0.003777,"RecordTime":"2021-08-27T10:34:44.804752Z"},"M1":{"Price":0.00352,"RecordTime":"2021-08-27T10:34:44.857364Z"},"M3":{"Price":0.00413,"RecordTime":"2021-08-27T10:34:44.903335Z"}},"PartitionKey":"jetwallet","RowKey":"LTCBTC","TimeStamp":"2021-08-27T10:34:45.0526146Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"LTCUSD","CurrentPrice":169.465,"H24P":0.06,"H24":{"Price":169.375,"RecordTime":"2021-08-20T11:28:37.5832338Z"},"D7":{"Price":189.9175,"RecordTime":"2021-08-27T10:34:44.953038Z"},"M1":{"Price":140.8762,"RecordTime":"2021-08-27T10:34:45.003068Z"},"M3":{"Price":144.5623,"RecordTime":"2021-08-27T10:34:45.04852Z"}},"PartitionKey":"jetwallet","RowKey":"LTCUSD","TimeStamp":"2021-08-27T10:34:45.2006129Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"TRXBTC","CurrentPrice":0.00000188,"H24P":0.0,"H24":{"Price":0.00000188,"RecordTime":"2021-08-20T11:28:37.6139046Z"},"D7":{"Price":0.0000018,"RecordTime":"2021-08-27T10:34:45.101142Z"},"M1":{"Price":0.00000174,"RecordTime":"2021-08-27T10:34:45.157366Z"},"M3":{"Price":0.00000174,"RecordTime":"2021-08-27T10:34:45.206864Z"}},"PartitionKey":"jetwallet","RowKey":"TRXBTC","TimeStamp":"2021-08-27T10:34:45.3565534Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"TRXUSD","CurrentPrice":0.084872,"H24P":0.07,"H24":{"Price":0.08482,"RecordTime":"2021-08-20T11:28:37.637955Z"},"D7":{"Price":0.09103,"RecordTime":"2021-08-27T10:34:45.256842Z"},"M1":{"Price":0.06087,"RecordTime":"2021-08-27T10:34:45.313602Z"},"M3":{"Price":0.087445,"RecordTime":"2021-08-27T10:34:45.369398Z"}},"PartitionKey":"jetwallet","RowKey":"TRXUSD","TimeStamp":"2021-08-27T10:34:45.5190682Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"XLMBTC","CurrentPrice":0.0,"H24P":0.0,"H24":{"Price":0.0,"RecordTime":"0001-01-01T00:00:00"},"D7":{"Price":0.00000764,"RecordTime":"2021-08-27T10:34:45.420089Z"},"M1":{"Price":0.00000764,"RecordTime":"2021-08-27T10:34:45.473258Z"},"M3":{"Price":0.00000764,"RecordTime":"2021-08-27T10:34:45.519656Z"}},"PartitionKey":"jetwallet","RowKey":"XLMBTC","TimeStamp":"2021-08-27T10:34:45.6729141Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"XLMUSD","CurrentPrice":0.34262,"H24P":0.09,"H24":{"Price":0.34234,"RecordTime":"2021-08-20T11:28:37.6616894Z"},"D7":{"Price":0.38436,"RecordTime":"2021-08-27T10:34:45.573804Z"},"M1":{"Price":0.27034,"RecordTime":"2021-08-27T10:34:45.62404Z"},"M3":{"Price":0.28439,"RecordTime":"2021-08-27T10:34:45.67416Z"}},"PartitionKey":"jetwallet","RowKey":"XLMUSD","TimeStamp":"2021-08-27T10:34:45.8274107Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"XRPBTC","CurrentPrice":0.0000245,"H24P":0.0,"H24":{"Price":0.0000245,"RecordTime":"2021-08-20T11:28:37.686023Z"},"D7":{"Price":0.00002541,"RecordTime":"2021-08-27T10:34:45.728023Z"},"M1":{"Price":0.00002,"RecordTime":"2021-08-27T10:34:45.779461Z"},"M3":{"Price":0.00002,"RecordTime":"2021-08-27T10:34:45.829896Z"}},"PartitionKey":"jetwallet","RowKey":"XRPBTC","TimeStamp":"2021-08-27T10:34:45.9793262Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"XRPUSD","CurrentPrice":1.106412,"H24P":0.08,"H24":{"Price":1.105625,"RecordTime":"2021-08-20T11:28:37.7116751Z"},"D7":{"Price":1.278075,"RecordTime":"2021-08-27T10:34:45.880169Z"},"M1":{"Price":0.73487,"RecordTime":"2021-08-27T10:34:45.927898Z"},"M3":{"Price":0.70801,"RecordTime":"2021-08-27T10:34:45.977608Z"}},"PartitionKey":"jetwallet","RowKey":"XRPUSD","TimeStamp":"2021-08-27T10:34:46.1271825Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"ZECBTC","CurrentPrice":0.0,"H24P":0.0,"H24":{"Price":0.0,"RecordTime":"0001-01-01T00:00:00"},"D7":{"Price":0.003264,"RecordTime":"2021-08-27T10:34:46.027258Z"},"M1":{"Price":0.003264,"RecordTime":"2021-08-27T10:34:46.07428Z"},"M3":{"Price":0.003264,"RecordTime":"2021-08-27T10:34:46.127362Z"}},"PartitionKey":"jetwallet","RowKey":"ZECBTC","TimeStamp":"2021-08-27T10:34:46.2770990Z","Expires":null},{"InstrumentPriceRecord":{"BrokerId":"jetwallet","InstrumentSymbol":"ZECUSD","CurrentPrice":140.16,"H24P":0.08,"H24":{"Price":140.06,"RecordTime":"2021-08-20T11:28:37.7356714Z"},"D7":{"Price":164.13,"RecordTime":"2021-08-27T10:34:46.177635Z"},"M1":{"Price":106.17,"RecordTime":"2021-08-27T10:34:46.229316Z"},"M3":{"Price":95.42,"RecordTime":"2021-08-27T10:34:46.278705Z"}},"PartitionKey":"jetwallet","RowKey":"ZECUSD","TimeStamp":"2021-08-27T10:34:46.4240405Z","Expires":null}] \ No newline at end of file diff --git a/test/test_case_2.json b/test/test_case_2.json deleted file mode 100644 index 208afd5..0000000 --- a/test/test_case_2.json +++ /dev/null @@ -1 +0,0 @@ -[{"DateTime":"2021-04-14T07:00:00Z","Event":" HICP (YoY)","Country":"ESP","Impact":"Low","Prev":"1.2%","Consensus":"1.2%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"00c15f98-6a7f-4e62-ac96-a4f371b9c009","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2378284Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Industrial Production (MoM)","Country":"GBR","Impact":"Medium","Prev":"-1.5%","Consensus":"0.5%","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"011c84aa-6fa9-45b5-bd70-244dd3ca9482","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2400480Z"},{"DateTime":"2021-04-15T01:30:00Z","Event":" Employment Change s.a.","Country":"AUS","Impact":"High","Prev":"88.70K","Consensus":"35.0K","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"0367c604-911e-4111-a0c8-548afdfd268d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2382962Z"},{"DateTime":"2021-04-14T12:30:00Z","Event":" Import Price Index (MoM)","Country":"USA","Impact":"Medium","Prev":"1.3%","Consensus":"0.9%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"046c07cf-8822-4629-8db7-f287cf5d3cb0","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2405703Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Wholesale Price Index (YoY)","Country":"DEU","Impact":"Low","Prev":"2.3%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"099d2fb7-2d8b-4292-a419-7d70c565d10e","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2401172Z"},{"DateTime":"2021-04-16T00:00:00Z","Event":" Eurogroup Meeting","Country":"EUR","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"09e7516a-00b7-461f-993e-7c9421b2ae82","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2380032Z"},{"DateTime":"2021-04-15T06:00:00Z","Event":" Harmonized Index of Consumer Prices (YoY)","Country":"DEU","Impact":"High","Prev":"2.0%","Consensus":"2.0%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"10a8e22b-9b01-4c52-b075-b7c6d149e406","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2401723Z"},{"DateTime":"2021-04-15T12:30:00Z","Event":" Retail Sales Control Group","Country":"USA","Impact":"High","Prev":"-3.5%","Consensus":"-0.9%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"113bf006-4f34-4912-95fb-38be383d41fd","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2407079Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Trade Balance; non-EU","Country":"GBR","Impact":"Low","Prev":"-£1.76B","Consensus":"£-4.95B","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"12f6b147-6029-4464-a2f7-233582edad41","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2407256Z"},{"DateTime":"2021-04-13T18:30:00Z","Event":" Mahavir Jayanti ","Country":"IND","Impact":"None","Prev":"","Consensus":"","Actual":"","Currency":"INR","PartitionKey":"mn","RowKey":"136fe380-2517-4b4b-b618-0db7c37eab38","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2386477Z"},{"DateTime":"2021-04-12T12:00:00Z","Event":" Cumulative Industrial Output","Country":"IND","Impact":"Low","Prev":"-12.2%","Consensus":"","Actual":"","Currency":"INR","PartitionKey":"mn","RowKey":"139bcf7d-2e73-4403-b861-88442c0e3a98","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2371207Z"},{"DateTime":"2021-04-14T00:00:00Z","Event":" Gross Domestic Product (QoQ)","Country":"SGP","Impact":"Low","Prev":"15.9%","Consensus":"","Actual":"","Currency":"SGD","PartitionKey":"mn","RowKey":"1a55973f-41b7-411c-8799-2ff69d0e656b","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2374288Z"},{"DateTime":"2021-04-15T14:00:00Z","Event":" NAHB Housing Market Index","Country":"USA","Impact":"Low","Prev":"82.00","Consensus":"83.0","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"1aaf4342-10c7-4456-b52c-ce3d815aaf13","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2393700Z"},{"DateTime":"2021-04-16T02:00:00Z","Event":" Gross Domestic Product (QoQ)","Country":"CHN","Impact":"High","Prev":"2.6%","Consensus":"1.5%","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"1b97dc89-ceec-4141-8b63-e7b9e9297455","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2376740Z"},{"DateTime":"2021-04-12T22:45:00Z","Event":" Electronic Card Retail Sales (YoY)","Country":"NZL","Impact":"Low","Prev":"-5.3%","Consensus":"","Actual":"","Currency":"NZD","PartitionKey":"mn","RowKey":"1cc83ed2-9bc4-4d9f-85d1-cc544256043e","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2381312Z"},{"DateTime":"2021-04-16T17:00:00Z","Event":" Baker Hughes US Oil Rig Count","Country":"USA","Impact":"Low","Prev":"337.00","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"1cebc74c-8992-4610-8c77-2d3550f3df58","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2367393Z"},{"DateTime":"2021-04-12T12:00:00Z","Event":" Industrial Output","Country":"IND","Impact":"Low","Prev":"-1.6%","Consensus":"-3.0%","Actual":"","Currency":"INR","PartitionKey":"mn","RowKey":"1e09c2f3-9f39-4c7f-916f-5818489dec5a","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2392833Z"},{"DateTime":"2021-04-15T05:00:00Z","Event":" Gross Domestic Product (YoY)","Country":"FIN","Impact":"Low","Prev":"-3.7%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"1e179d16-9c20-473c-9978-e72c42423a03","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2369230Z"},{"DateTime":"2021-04-12T18:00:00Z","Event":" Monthly Budget Statement","Country":"USA","Impact":"Medium","Prev":"-$311.00B","Consensus":"$-265.0B","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"1e30fd9a-1f82-48ff-aaed-c9ce244bbff1","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2406021Z"},{"DateTime":"2021-04-13T02:00:00Z","Event":" Trade Balance USD","Country":"CHN","Impact":"Medium","Prev":"$103.25B","Consensus":"$52.55B","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"1ee2f75c-4c09-4d89-afdf-1840f168a6a5","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2399418Z"},{"DateTime":"2021-04-12T06:00:00Z","Event":" Inflation (HICP) (YoY)","Country":"DNK","Impact":"Low","Prev":"0.5%","Consensus":"","Actual":"0.9%","Currency":"DKK","PartitionKey":"mn","RowKey":"20282fed-8051-405f-baf3-b691845d5483","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2385945Z"},{"DateTime":"2021-04-16T07:00:00Z","Event":" HICP (MoM)","Country":"AUT","Impact":"Low","Prev":"0.3%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"2031f2f1-3923-4fd0-9123-ce5bcf996abe","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2381643Z"},{"DateTime":"2021-04-15T13:15:00Z","Event":" Industrial Production (MoM)","Country":"USA","Impact":"Low","Prev":"-2.2%","Consensus":"2.8%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"2090cc4e-2213-4888-b510-4a7e82a34211","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2396824Z"},{"DateTime":"2021-04-15T08:30:00Z","Event":" BoE Credit Conditions Survey","Country":"GBR","Impact":"Low","Prev":"","Consensus":"","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"21a15a8e-a660-4613-b634-2c4119cc2c61","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2396150Z"},{"DateTime":"2021-04-15T12:30:00Z","Event":" Continuing Jobless Claims","Country":"USA","Impact":"Medium","Prev":"3.73M","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"2241f296-46a3-49dd-af3f-1491a211687d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2381803Z"},{"DateTime":"2021-04-15T06:00:00Z","Event":" Trade Balance","Country":"NOR","Impact":"Low","Prev":"25.10B","Consensus":"","Actual":"","Currency":"NOK","PartitionKey":"mn","RowKey":"2306fe42-0c97-4c12-b5a1-4a7ff2d45562","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2392138Z"},{"DateTime":"2021-04-15T14:30:00Z","Event":" EIA Natural Gas Storage Change","Country":"USA","Impact":"Low","Prev":"20.00B","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"241c270c-ccdd-4008-9996-43f35e4ad7e0","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2395481Z"},{"DateTime":"2021-04-16T16:00:00Z","Event":" Producer Price Index (YoY)","Country":"RUS","Impact":"Low","Prev":"10.7%","Consensus":"","Actual":"","Currency":"RUB","PartitionKey":"mn","RowKey":"24f8f613-68b2-4161-946c-799039315c8c","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2397649Z"},{"DateTime":"2021-04-14T23:50:00Z","Event":" Foreign Investment in Japan Stocks","Country":"JPN","Impact":"Low","Prev":"¥978.40B","Consensus":"","Actual":"","Currency":"JPY","PartitionKey":"mn","RowKey":"252c4a3d-a19e-4348-bf9b-6c9a1fc15bfb","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2409468Z"},{"DateTime":"2021-04-16T09:00:00Z","Event":" Consumer Price Index (YoY)","Country":"EUR","Impact":"Low","Prev":"1.3%","Consensus":"1.3%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"257eef8e-1b2b-413f-a52c-4aed152b20f4","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2371553Z"},{"DateTime":"2021-04-15T06:45:00Z","Event":" Consumer Price Index (EU norm) (YoY)","Country":"FRA","Impact":"Low","Prev":"1.4%","Consensus":"1.4%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"2a91c022-ce35-4e4d-aa87-0dd9651e5143","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2394659Z"},{"DateTime":"2021-04-13T01:30:00Z","Event":" National Australia Bank's Business Confidence","Country":"AUS","Impact":"Low","Prev":"16.00","Consensus":"","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"2d758184-f042-4dc2-9733-cf7124d11f07","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2403047Z"},{"DateTime":"2021-04-13T12:00:00Z","Event":" Retail Sales (MoM)","Country":"BRA","Impact":"Low","Prev":"-0.2%","Consensus":"-0.3%","Actual":"","Currency":"BRL","PartitionKey":"mn","RowKey":"301e7dd0-b8bc-46eb-b3e3-5397da7c52e4","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2380487Z"},{"DateTime":"2021-04-13T21:00:00Z","Event":" Export Price Growth (YoY)","Country":"KOR","Impact":"Low","Prev":"0.2%","Consensus":"0.6%","Actual":"","Currency":"KRW","PartitionKey":"mn","RowKey":"30badf36-2596-4753-ba99-12bd3eb5409d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2409655Z"},{"DateTime":"2021-04-15T06:45:00Z","Event":" Consumer Price Index (EU norm) (MoM)","Country":"FRA","Impact":"Low","Prev":"0.7%","Consensus":"0.7%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"32d7eb25-38c6-4f3e-9943-182ead5b0784","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2373735Z"},{"DateTime":"2021-04-14T01:00:00Z","Event":" RBNZ Rate Statement","Country":"NZL","Impact":"High","Prev":"","Consensus":"","Actual":"","Currency":"NZD","PartitionKey":"mn","RowKey":"339d97c7-806c-410b-a782-89ce7e25daf1","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2382291Z"},{"DateTime":"2021-04-15T15:00:00Z","Event":" Industrial output (YoY)","Country":"COL","Impact":"Low","Prev":"-1.6%","Consensus":"","Actual":"","Currency":"COP","PartitionKey":"mn","RowKey":"339e2a24-2287-4740-81ee-a85c5074d257","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2401892Z"},{"DateTime":"2021-04-15T01:30:00Z","Event":" Part-Time Employment","Country":"AUS","Impact":"Low","Prev":"-0.50K","Consensus":"","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"34d8945b-4f1b-497e-b7f5-f6d294421a8b","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2408589Z"},{"DateTime":"2021-04-16T01:30:00Z","Event":" House Price Index ","Country":"CHN","Impact":"Low","Prev":"4.3%","Consensus":"","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"36e226b9-a87e-471b-846f-90f4d945af26","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2385774Z"},{"DateTime":"2021-04-15T08:00:00Z","Event":" Consumer Price Index (YoY)","Country":"ITA","Impact":"Low","Prev":"0.8%","Consensus":"0.6%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"37cd7d3c-d478-43d6-bb97-c9084c6c571c","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2379231Z"},{"DateTime":"2021-04-16T19:30:00Z","Event":" CFTC AUD NC Net Positions","Country":"AUS","Impact":"Low","Prev":"$4.10K","Consensus":"","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"39b85f20-f86f-4133-90b7-0f032b705244","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2405552Z"},{"DateTime":"2021-04-15T14:00:00Z","Event":" Business Inventories","Country":"USA","Impact":"Low","Prev":"0.3%","Consensus":"0.4%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"39d51349-be3b-43df-8bf0-1c838ca8aa32","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2401528Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Wholesale Price Index (MoM)","Country":"DEU","Impact":"Low","Prev":"1.4%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"3aa9d835-0b4d-4693-8097-7817ed143552","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2407549Z"},{"DateTime":"2021-04-12T07:00:00Z","Event":" 3mth quarterly jobless average","Country":"TUR","Impact":"Low","Prev":"14.2%","Consensus":"","Actual":"13.4%","Currency":"TRY","PartitionKey":"mn","RowKey":"3bf298f9-7b87-481e-93aa-ab54668940b7","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2406338Z"},{"DateTime":"2021-04-14T07:00:00Z","Event":" Consumer Price Index (YoY)","Country":"ESP","Impact":"Low","Prev":"1.3%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"3c724157-d5cb-49c2-940c-ad0ab3232a2f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2385607Z"},{"DateTime":"2021-04-16T09:00:00Z","Event":" Trade Balance s.a.","Country":"EUR","Impact":"Low","Prev":"€24.20B","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"3d338244-b19c-4818-99e8-7dff35b4f8f6","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2378740Z"},{"DateTime":"2021-04-15T12:30:00Z","Event":" Retail Sales (MoM)","Country":"USA","Impact":"High","Prev":"-3.0%","Consensus":"5.5%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"3e4a070f-9512-4d51-83b5-b59ab587320f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2411305Z"},{"DateTime":"2021-04-16T12:30:00Z","Event":" Wholesale Sales (MoM)","Country":"CAN","Impact":"Medium","Prev":"4.0%","Consensus":"5.0%","Actual":"","Currency":"CAD","PartitionKey":"mn","RowKey":"41fc8c24-1891-4198-a40f-cf1845021fc4","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2408073Z"},{"DateTime":"2021-04-13T17:00:00Z","Event":" 30-Year Bond Auction","Country":"USA","Impact":"Low","Prev":"2.295%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"4286a246-bf85-4ead-acdc-a1afa7a1871e","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2406169Z"},{"DateTime":"2021-04-16T02:00:00Z","Event":" Retail Sales (YoY)","Country":"CHN","Impact":"High","Prev":"33.8%","Consensus":"27.2%","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"42ce5f22-44ea-419d-9624-28c68f2fc18c","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2410946Z"},{"DateTime":"2021-04-16T21:00:00Z","Event":" Orthodox Good Friday ","Country":"GRC","Impact":"None","Prev":"","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"42f7b871-a3a5-4125-b327-2e3847be2e80","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2375806Z"},{"DateTime":"2021-04-13T03:00:00Z","Event":" Money Supply Growth","Country":"KOR","Impact":"Low","Prev":"8.9%","Consensus":"8.6%","Actual":"","Currency":"KRW","PartitionKey":"mn","RowKey":"4355c6ca-efa0-42b3-bd10-9e6133eda6d7","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2394974Z"},{"DateTime":"2021-04-13T07:00:00Z","Event":" Consumer Price Index (MoM)","Country":"CZE","Impact":"Low","Prev":"0.2%","Consensus":"0.3%","Actual":"","Currency":"CZK","PartitionKey":"mn","RowKey":"4445c49c-3ff4-4ced-bff6-51c6d7d62934","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2406809Z"},{"DateTime":"2021-04-13T09:00:00Z","Event":" ZEW Survey – Current Situation","Country":"DEU","Impact":"Medium","Prev":"-61.00","Consensus":"-52.0","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"4729a55a-da36-4242-a7c1-ab836ebef8ce","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2375029Z"},{"DateTime":"2021-04-16T08:00:00Z","Event":" Global Trade Balance","Country":"ITA","Impact":"Low","Prev":"€1.59B","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"4751686e-9925-4a7f-b4be-f4f36ea1cc15","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2369062Z"},{"DateTime":"2021-04-15T09:00:00Z","Event":" Unemployment Rate (MoM)","Country":"GRC","Impact":"Low","Prev":"15.8%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"47a98ae1-781a-414d-b04c-bdefff520112","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2398711Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Total Trade Balance","Country":"GBR","Impact":"Low","Prev":"-£1.63B","Consensus":"","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"499ff415-2fb7-4e5a-ae6a-aacddb23e5c0","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2376574Z"},{"DateTime":"2021-04-14T21:45:00Z","Event":" Food Price Index (MoM)","Country":"NZL","Impact":"Low","Prev":"-0.9%","Consensus":"","Actual":"","Currency":"NZD","PartitionKey":"mn","RowKey":"4a707ee6-32da-4207-8cc9-806ad2c88d25","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2369770Z"},{"DateTime":"2021-04-15T11:00:00Z","Event":" CBRT Interest Rate Decision","Country":"TUR","Impact":"Low","Prev":"19.0%","Consensus":"19.0%","Actual":"","Currency":"TRY","PartitionKey":"mn","RowKey":"4b702627-568c-4edc-b517-7254f4bfcff5","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2379871Z"},{"DateTime":"2021-04-16T02:00:00Z","Event":" Industrial Production (YoY)","Country":"CHN","Impact":"Medium","Prev":"35.1%","Consensus":"15.6%","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"4c1e3a1c-4ea9-4760-bd9c-3a792e7ffd2f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2404465Z"},{"DateTime":"2021-04-11T23:50:00Z","Event":" Bank Lending (YoY)","Country":"JPN","Impact":"Low","Prev":"6.2%","Consensus":"","Actual":"6.3%","Currency":"JPY","PartitionKey":"mn","RowKey":"4d93e6bf-857f-4ece-8002-3ec59de90091","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2391482Z"},{"DateTime":"2021-04-16T09:00:00Z","Event":" Consumer Price Index (MoM)","Country":"EUR","Impact":"Medium","Prev":"0.2%","Consensus":"0.9%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"4e835154-9bcc-4852-852c-5a50f2ae4412","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2390054Z"},{"DateTime":"2021-04-12T09:00:00Z","Event":" Retail Sales (YoY)","Country":"EUR","Impact":"High","Prev":"-6.4%","Consensus":"-5.4%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"4eb8feba-de02-45ce-9aef-da73109bae3b","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2390982Z"},{"DateTime":"2021-04-13T12:55:00Z","Event":" Redbook Index (YoY)","Country":"USA","Impact":"Low","Prev":"10.6%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"5007a773-8bbc-4d9b-b36a-46a296c7e299","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2407903Z"},{"DateTime":"2021-04-13T08:40:00Z","Event":" 6-Month Letras Auction","Country":"ESP","Impact":"Low","Prev":"-0.524%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"50857da4-4a4b-4179-8fad-4bad83774948","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2370799Z"},{"DateTime":"2021-04-16T19:30:00Z","Event":" CFTC Oil NC Net Positions","Country":"USA","Impact":"Low","Prev":"511.70K","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"50906583-f04b-40d6-87f1-5da7dd8f30e1","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2373401Z"},{"DateTime":"2021-04-14T07:00:00Z","Event":" Headline Inflation (MoM)","Country":"SVK","Impact":"Low","Prev":"0.3%","Consensus":"0.4%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"51efcb0a-a909-4618-ba68-a82e2d2455e5","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2397156Z"},{"DateTime":"2021-04-15T15:00:00Z","Event":" Retail Sales (YoY)","Country":"COL","Impact":"Low","Prev":"-6.4%","Consensus":"","Actual":"","Currency":"COP","PartitionKey":"mn","RowKey":"53a17201-3964-4e6e-9410-64c6fc608d21","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2388376Z"},{"DateTime":"2021-04-14T07:00:00Z","Event":" Consumer Price Index (MoM)","Country":"ESP","Impact":"Low","Prev":"1.0%","Consensus":"-0.6%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"551e0f7e-2e96-432f-b39c-9a5bdc21f076","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2408416Z"},{"DateTime":"2021-04-15T08:00:00Z","Event":" Consumer Price Index (EU Norm) (MoM)","Country":"ITA","Impact":"Low","Prev":"1.8%","Consensus":"1.8%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"564ffbd2-c3b2-4a87-b32e-52e0f124ad8d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2367878Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Manufacturing Production (MoM)","Country":"GBR","Impact":"Medium","Prev":"-2.3%","Consensus":"-0.8%","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"57b5866a-883d-4825-b442-4c46f6fd1d41","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2381476Z"},{"DateTime":"2021-04-14T07:00:00Z","Event":" Core Inflation (YoY)","Country":"SVK","Impact":"Low","Prev":"1.3%","Consensus":"1.8%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"586b89c6-3db9-48e9-91cb-abedee14ce43","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2373913Z"},{"DateTime":"2021-04-14T00:00:00Z","Event":" Gross Domestic Product (YoY)","Country":"SGP","Impact":"Low","Prev":"-2.4%","Consensus":"-0.2%","Actual":"","Currency":"SGD","PartitionKey":"mn","RowKey":"59f40c29-1787-4b0c-8cef-689966b939eb","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2372700Z"},{"DateTime":"2021-04-16T08:00:00Z","Event":" Trade Balance EU","Country":"ITA","Impact":"Low","Prev":"-€0.19B","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"5c0601e3-d009-40a7-a396-0fe31a24a915","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2404612Z"},{"DateTime":"2021-04-15T20:00:00Z","Event":" Total Net TIC Flows","Country":"USA","Impact":"Low","Prev":"$106.30B","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"5c06ddaf-9de5-4614-97b7-b3335fe55115","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2400650Z"},{"DateTime":"2021-04-14T12:30:00Z","Event":" Export Price Index (YoY)","Country":"USA","Impact":"Low","Prev":"5.2%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"5d13bb37-861e-4742-8510-3ff241e0c088","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2386295Z"},{"DateTime":"2021-04-12T07:00:00Z","Event":" M2 Money Supply (YoY)","Country":"CHN","Impact":"Low","Prev":"10.1%","Consensus":"9.6%","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"5ee6addc-419e-4aad-af40-f1c06497cecb","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2371743Z"},{"DateTime":"2021-04-14T07:00:00Z","Event":" ECB's De Guindos speech","Country":"EUR","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"5f65e81d-0344-42ca-ab0b-dfccca6b0bb9","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2396662Z"},{"DateTime":"2021-04-12T17:00:00Z","Event":" 10-Year Note Auction","Country":"USA","Impact":"Low","Prev":"1.523%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"61d06b7f-0fa5-41a7-ae28-8e670011d92d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2389280Z"},{"DateTime":"2021-04-16T19:30:00Z","Event":" CFTC JPY NC Net Positions","Country":"JPN","Impact":"Low","Prev":"-¥58.00K","Consensus":"","Actual":"","Currency":"JPY","PartitionKey":"mn","RowKey":"63831b73-8056-4200-a6a8-d6e2e46593f1","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2403222Z"},{"DateTime":"2021-04-15T12:30:00Z","Event":" Philadelphia Fed Manufacturing Survey","Country":"USA","Impact":"Medium","Prev":"51.80","Consensus":"43.0","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"63f17399-170f-4acd-ae22-3b91fc4dcf58","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2393191Z"},{"DateTime":"2021-04-14T18:30:00Z","Event":" Fed's Williams speech","Country":"USA","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"65f1b63f-0b6b-4fe6-9b56-eeba337425b0","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2377513Z"},{"DateTime":"2021-04-15T02:00:00Z","Event":" Trade Balance","Country":"KOR","Impact":"Low","Prev":"$4.17B","Consensus":"","Actual":"","Currency":"KRW","PartitionKey":"mn","RowKey":"66c6daac-54f3-4809-a16b-86121e9aa43c","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2391151Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Consumer Price Index","Country":"ROU","Impact":"Low","Prev":"3.2%","Consensus":"","Actual":"","Currency":"ROL","PartitionKey":"mn","RowKey":"66dcc8d2-84c0-43f9-88a3-142a073019ed","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2388667Z"},{"DateTime":"2021-04-13T09:30:00Z","Event":" 10-y Bond Auction","Country":"DEU","Impact":"Medium","Prev":"-0.36%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"682b186c-7e05-478a-b1aa-25c31f57dfa1","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2373564Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Manufacturing Production (YoY)","Country":"GBR","Impact":"Medium","Prev":"-5.2%","Consensus":"-5.1%","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"68841303-4065-4787-a56c-a8328f7ac87d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2372348Z"},{"DateTime":"2021-04-14T07:30:00Z","Event":" Consumer Price Index (YoY)","Country":"SWE","Impact":"Low","Prev":"1.4%","Consensus":"1.7%","Actual":"","Currency":"SEK","PartitionKey":"mn","RowKey":"68b29268-5fe9-4b04-b258-ec6ee8e8c147","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2386649Z"},{"DateTime":"2021-04-15T21:00:00Z","Event":" REINZ House Price Index (MoM)","Country":"NZL","Impact":"Medium","Prev":"5.2%","Consensus":"","Actual":"","Currency":"NZD","PartitionKey":"mn","RowKey":"6b3faf87-52fa-4f6b-9634-c90adf7492e3","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2381965Z"},{"DateTime":"2021-04-15T22:30:00Z","Event":" Business NZ PMI","Country":"NZL","Impact":"Medium","Prev":"53.40","Consensus":"51.3","Actual":"","Currency":"NZD","PartitionKey":"mn","RowKey":"6ba58366-22c8-49e8-9380-a88f07a30465","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2376426Z"},{"DateTime":"2021-04-14T23:50:00Z","Event":" Foreign Bond Investment","Country":"JPN","Impact":"Low","Prev":"¥377.00B","Consensus":"","Actual":"","Currency":"JPY","PartitionKey":"mn","RowKey":"70543862-4db8-446f-a3b9-5f450609c227","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2390824Z"},{"DateTime":"2021-04-12T05:00:00Z","Event":" Current Account","Country":"FIN","Impact":"Low","Prev":"-€0.10B","Consensus":"","Actual":"€0.40B","Currency":"EUR","PartitionKey":"mn","RowKey":"708588cf-a3b4-492d-8211-4bd8ab31b4fd","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2397823Z"},{"DateTime":"2021-04-14T09:00:00Z","Event":" Industrial Production s.a. (MoM)","Country":"EUR","Impact":"Medium","Prev":"0.8%","Consensus":"0.5%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"714c63fd-7892-4ae3-a026-da4b60a06b5d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2379719Z"},{"DateTime":"2021-04-14T14:00:00Z","Event":" BoE's Haskel speech","Country":"GBR","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"71fc49f8-a6c9-45ba-9f62-6b75e17870f5","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2410412Z"},{"DateTime":"2021-04-11T23:50:00Z","Event":" Producer Price Index (YoY)","Country":"JPN","Impact":"Low","Prev":"-0.6%","Consensus":"0.5%","Actual":"1.0%","Currency":"JPY","PartitionKey":"mn","RowKey":"7648b986-08da-4b76-b27c-8cca23823c9b","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2377203Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Index of Services (3M/3M)","Country":"GBR","Impact":"Low","Prev":"-2.4%","Consensus":"","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"7913d0be-25e4-42c1-9e3f-0e52e443d44e","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2390218Z"},{"DateTime":"2021-04-14T07:30:00Z","Event":" Consumer Price Index (MoM)","Country":"SWE","Impact":"Low","Prev":"0.3%","Consensus":"","Actual":"","Currency":"SEK","PartitionKey":"mn","RowKey":"7ab20276-661c-4748-878c-27dd2b79684d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2398155Z"},{"DateTime":"2021-04-13T08:00:00Z","Event":" Industrial Output s.a. (MoM)","Country":"ITA","Impact":"Low","Prev":"1.0%","Consensus":"0.6%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"7b62e2c3-7268-447b-b71a-619a1894f27d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2368073Z"},{"DateTime":"2021-04-14T05:00:00Z","Event":" Consumer Price Index (YoY)","Country":"FIN","Impact":"Low","Prev":"0.9%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"7c0646a6-3340-4f61-ba05-4b026d560429","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2405242Z"},{"DateTime":"2021-04-15T13:00:00Z","Event":" Industrial Output","Country":"RUS","Impact":"Low","Prev":"-3.7%","Consensus":"-1.0%","Actual":"","Currency":"RUB","PartitionKey":"mn","RowKey":"7d0fa610-2109-4eee-8e54-004a0928c41d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2406489Z"},{"DateTime":"2021-04-14T06:30:00Z","Event":" WPI Inflation","Country":"IND","Impact":"Low","Prev":"4.17%","Consensus":"5.9%","Actual":"","Currency":"INR","PartitionKey":"mn","RowKey":"7d32ed93-c7e9-4f39-bc70-cc3544762ca9","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2386128Z"},{"DateTime":"2021-04-14T01:00:00Z","Event":" RBNZ Interest Rate Decision","Country":"NZL","Impact":"High","Prev":"0.25%","Consensus":"0.25%","Actual":"","Currency":"NZD","PartitionKey":"mn","RowKey":"7d9837f0-2e58-43df-b484-26770149b6fd","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2375342Z"},{"DateTime":"2021-04-16T19:30:00Z","Event":" CFTC Gold NC Net Positions","Country":"USA","Impact":"Low","Prev":"$189.50K","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"813c48e9-eefc-46f7-a51e-da1fa4cd5ba3","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2399607Z"},{"DateTime":"2021-04-13T01:00:00Z","Event":" HIA New Home Sales (MoM)","Country":"AUS","Impact":"Medium","Prev":"22.9%","Consensus":"","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"8142917e-d6c4-4cce-a75f-e6c46c3ed404","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2382624Z"},{"DateTime":"2021-04-12T17:00:00Z","Event":" 3-Year Note Auction","Country":"USA","Impact":"Low","Prev":"0.355%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"8339b53f-316e-4c4a-b45f-f1de77a95da6","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2404299Z"},{"DateTime":"2021-04-14T07:00:00Z","Event":" Headline Inflation (YoY)","Country":"SVK","Impact":"Low","Prev":"0.9%","Consensus":"1.4%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"85461f58-93d1-48dc-91e9-37e4643e22d6","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2395148Z"},{"DateTime":"2021-04-12T23:01:00Z","Event":" BRC Like-For-Like Retail Sales (YoY)","Country":"GBR","Impact":"Medium","Prev":"9.5%","Consensus":"","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"8764b15e-f342-4948-87cf-551fede3b773","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2386958Z"},{"DateTime":"2021-04-13T02:00:00Z","Event":" Imports (YoY)","Country":"CHN","Impact":"Medium","Prev":"22.2%","Consensus":"21.6%","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"87ca8fe8-2ca5-4025-b518-b07ae094abe1","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2395305Z"},{"DateTime":"2021-04-16T09:00:00Z","Event":" Trade Balance n.s.a.","Country":"EUR","Impact":"Low","Prev":"€6.30B","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"889d9600-cc5b-46cf-8a0d-d9379416ef85","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2393371Z"},{"DateTime":"2021-04-15T01:30:00Z","Event":" Fulltime Employment","Country":"AUS","Impact":"Medium","Prev":"89.10K","Consensus":"","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"8b208c7f-ddc8-4722-8b7b-ad03fc75a5c3","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2389594Z"},{"DateTime":"2021-04-13T12:30:00Z","Event":" Consumer Price Index (YoY)","Country":"USA","Impact":"Medium","Prev":"1.7%","Consensus":"2.5%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"8b45fdc1-fbd6-487e-ae87-ae74f5600d27","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2372176Z"},{"DateTime":"2021-04-16T19:30:00Z","Event":" CFTC EUR NC Net Positions","Country":"EUR","Impact":"Low","Prev":"€67.50K","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"8b78c511-7984-4232-93b4-da16a5a140e2","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2387124Z"},{"DateTime":"2021-04-12T15:30:00Z","Event":" 6-Month Bill Auction","Country":"USA","Impact":"Low","Prev":"0.035%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"8de5fe86-8f30-4448-aef7-129ec1fe6927","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2381129Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Industrial Production (MoM)","Country":"ROU","Impact":"Low","Prev":"0.0%","Consensus":"","Actual":"","Currency":"ROL","PartitionKey":"mn","RowKey":"8df03383-50bc-46d3-b78f-e5c0099aebb5","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2408944Z"},{"DateTime":"2021-04-13T21:45:00Z","Event":" Visitor Arrivals (YoY)","Country":"NZL","Impact":"Low","Prev":"-98.7%","Consensus":"","Actual":"","Currency":"NZD","PartitionKey":"mn","RowKey":"8ec03f02-15de-4bd5-bf11-63e0013a2fdb","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2372531Z"},{"DateTime":"2021-04-15T12:30:00Z","Event":" NY Empire State Manufacturing Index","Country":"USA","Impact":"Low","Prev":"17.40","Consensus":"18.2","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"8ffcf454-c68a-4c12-a1f2-f153d7353aaf","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2374581Z"},{"DateTime":"2021-04-13T08:00:00Z","Event":" Industrial Output w.d.a (YoY)","Country":"ITA","Impact":"Low","Prev":"-2.4%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"90b8da2a-ab99-41af-8de6-a205f1f30a45","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2379066Z"},{"DateTime":"2021-04-15T06:00:00Z","Event":" Consumer Price Index (YoY)","Country":"DEU","Impact":"Low","Prev":"1.7%","Consensus":"1.7%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"93bcd121-a5c0-4021-99a1-9b421f345ea6","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2370454Z"},{"DateTime":"2021-04-13T12:30:00Z","Event":" Consumer Price Index (MoM)","Country":"USA","Impact":"Medium","Prev":"0.4%","Consensus":"0.5%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"9419f2b5-6b56-4fe2-b548-d1524230847b","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2410012Z"},{"DateTime":"2021-04-14T11:00:00Z","Event":" Retail Sales (YoY)","Country":"ZAF","Impact":"Low","Prev":"-3.5%","Consensus":"","Actual":"","Currency":"ZAR","PartitionKey":"mn","RowKey":"94d04644-8ad7-4d41-92e5-66c4a16a8637","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2377349Z"},{"DateTime":"2021-04-14T16:00:00Z","Event":" Fed's Chair Powell speech","Country":"USA","Impact":"High","Prev":"","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"964afd11-2cbf-493c-b00d-ca8dc0cf19cb","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2413428Z"},{"DateTime":"2021-04-15T06:45:00Z","Event":" Inflation ex-tobacco (MoM)","Country":"FRA","Impact":"Low","Prev":"0.0%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"992424ac-fe06-4069-b82b-65db0a6f5dd1","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2380181Z"},{"DateTime":"2021-04-13T20:30:00Z","Event":" API Weekly Crude Oil Stock","Country":"USA","Impact":"Low","Prev":"-2.62M","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"9925be1a-2ec6-4518-9317-8e71c2dd268e","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2376264Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Goods Trade Balance","Country":"GBR","Impact":"Low","Prev":"-£9.83B","Consensus":"£-10.6B","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"992c218d-1aa1-4a0c-8a25-9c559cf2278c","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2371039Z"},{"DateTime":"2021-04-12T13:00:00Z","Event":" BoE's Tenreyro speech","Country":"GBR","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"9a555ec5-55b1-4c6a-81d1-08a4665d9eae","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2378592Z"},{"DateTime":"2021-04-15T15:30:00Z","Event":" 4-Week Bill Auction","Country":"USA","Impact":"Low","Prev":"0.01%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"9b36bc69-33a5-4aea-bc5b-b3382f99f8ac","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2397311Z"},{"DateTime":"2021-04-15T13:00:00Z","Event":" Central Bank Reserves $","Country":"RUS","Impact":"Low","Prev":"$574.80B","Consensus":"","Actual":"","Currency":"RUB","PartitionKey":"mn","RowKey":"9b742027-ad95-421d-883b-dd5a6d01bc22","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2391311Z"},{"DateTime":"2021-04-13T07:00:00Z","Event":" Consumer Price Index (YoY)","Country":"CZE","Impact":"Low","Prev":"2.1%","Consensus":"2.4%","Actual":"","Currency":"CZK","PartitionKey":"mn","RowKey":"9d01d1d4-4e01-435c-9353-3c6110165cf5","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2404761Z"},{"DateTime":"2021-04-13T08:00:00Z","Event":" Current Account (MoM)","Country":"CZE","Impact":"Low","Prev":"28.65B","Consensus":"","Actual":"","Currency":"CZK","PartitionKey":"mn","RowKey":"9e08b25e-c509-4d07-8341-4e781c1c23c3","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2403542Z"},{"DateTime":"2021-04-12T15:30:00Z","Event":" 3-Month Bill Auction","Country":"USA","Impact":"Low","Prev":"0.02%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"9e1fe5c9-06ed-4d74-b524-a4c797f0782f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2399936Z"},{"DateTime":"2021-04-15T13:15:00Z","Event":" Capacity Utilization","Country":"USA","Impact":"Low","Prev":"73.8%","Consensus":"75.6%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"9e47d4d8-397b-4e08-a872-6f5fc000113a","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2400819Z"},{"DateTime":"2021-04-13T10:00:00Z","Event":" Consumer Price Index (YoY)","Country":"PRT","Impact":"Low","Prev":"0.5%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"9f299026-5d48-4268-b88c-2996656e40af","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2393535Z"},{"DateTime":"2021-04-13T12:30:00Z","Event":" Consumer Price Index ex Food & Energy (MoM)","Country":"USA","Impact":"High","Prev":"0.1%","Consensus":"0.2%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"a0686800-5b6a-443a-abd3-41d5a65ca476","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2396334Z"},{"DateTime":"2021-04-16T09:00:00Z","Event":" BoE's Cunliffe speech","Country":"GBR","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"a0c532fd-48f6-4c8d-950b-0e3b317e674f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2391802Z"},{"DateTime":"2021-04-13T02:00:00Z","Event":" Exports (YoY) CNY","Country":"CHN","Impact":"Low","Prev":"139.5%","Consensus":"","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"a2509ffa-8257-433b-85f6-b0ecedb82f56","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2402754Z"},{"DateTime":"2021-04-12T14:30:00Z","Event":" Bank of Canada Business Outlook Survey","Country":"CAN","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"CAD","PartitionKey":"mn","RowKey":"a2b526c3-8a96-49aa-a016-21b84c1db73d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2375660Z"},{"DateTime":"2021-04-13T23:50:00Z","Event":" Machinery Orders (YoY)","Country":"JPN","Impact":"Low","Prev":"1.5%","Consensus":"2.3%","Actual":"","Currency":"JPY","PartitionKey":"mn","RowKey":"a3b401b4-3b70-4bb0-82b5-ebc674af429d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2370635Z"},{"DateTime":"2021-04-16T12:30:00Z","Event":" Building Permits Change","Country":"USA","Impact":"High","Prev":"-8.8%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"a4c71333-c720-4dcf-81b4-478a84c1c0a5","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2377824Z"},{"DateTime":"2021-04-16T07:00:00Z","Event":" EU Norm Inflation (YoY)","Country":"SVK","Impact":"Low","Prev":"0.9%","Consensus":"1.4%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"a56c6c79-a94c-4e0d-8a58-d9d8ac9a4975","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2373059Z"},{"DateTime":"2021-04-16T12:30:00Z","Event":" Foreign Portfolio Investment in Canadian Securities","Country":"CAN","Impact":"Low","Prev":"$1.27B","Consensus":"","Actual":"","Currency":"CAD","PartitionKey":"mn","RowKey":"a6bfaf23-51ab-47e8-8d93-f60ca8a3e2b8","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2372878Z"},{"DateTime":"2021-04-12T04:00:00Z","Event":" Retail Sales (YoY)","Country":"IDN","Impact":"Low","Prev":"-16.4%","Consensus":"","Actual":"-18.1%","Currency":"IDR","PartitionKey":"mn","RowKey":"a729cb9c-04f4-4f76-a7cb-027fe5f6dfcc","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2399066Z"},{"DateTime":"2021-04-15T12:30:00Z","Event":" Retail Sales ex Autos (MoM)","Country":"USA","Impact":"High","Prev":"-2.7%","Consensus":"4.8%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"aa75bd47-a75e-486f-b335-1293fdd9379e","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2387291Z"},{"DateTime":"2021-04-15T12:30:00Z","Event":" Initial Jobless Claims","Country":"USA","Impact":"High","Prev":"744.00K","Consensus":"700.0K","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"aa7c257a-81ab-4576-b24f-42c218c631c2","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2398430Z"},{"DateTime":"2021-04-15T04:00:00Z","Event":" Imports","Country":"IDN","Impact":"Low","Prev":"14.86%","Consensus":"","Actual":"","Currency":"IDR","PartitionKey":"mn","RowKey":"aac21b82-7ab1-4fbf-b133-bc2af2785163","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2377050Z"},{"DateTime":"2021-04-13T12:30:00Z","Event":" Consumer Price Index n.s.a (MoM)","Country":"USA","Impact":"Low","Prev":"263.01","Consensus":"263.08","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"ab1a94ac-c420-413b-83b9-ba0da3f4a7f9","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2395991Z"},{"DateTime":"2021-04-15T12:30:00Z","Event":" Manufacturing Sales (MoM)","Country":"CAN","Impact":"Low","Prev":"3.1%","Consensus":"","Actual":"","Currency":"CAD","PartitionKey":"mn","RowKey":"acee1be9-3e4c-4738-a518-1693919d016c","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2369590Z"},{"DateTime":"2021-04-14T06:15:00Z","Event":" BoJ's Governor Kuroda speech","Country":"JPN","Impact":"High","Prev":"","Consensus":"","Actual":"","Currency":"JPY","PartitionKey":"mn","RowKey":"ad6bc354-35dd-4ded-8174-460aabb8b678","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2392313Z"},{"DateTime":"2021-04-16T07:00:00Z","Event":" HICP (YoY)","Country":"AUT","Impact":"Low","Prev":"1.4%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"aded3a9b-1614-48e7-9e47-800530067f2e","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2399245Z"},{"DateTime":"2021-04-15T01:00:00Z","Event":" Consumer Inflation Expectations","Country":"AUS","Impact":"Medium","Prev":"4.1%","Consensus":"","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"ae14c6ac-c74b-41cc-9ce4-f0f9a3502512","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2394476Z"},{"DateTime":"2021-04-16T19:30:00Z","Event":" CFTC S&P 500 NC Net Positions","Country":"USA","Impact":"Low","Prev":"-$45.30K","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"af95bf3f-73d9-4a25-b62f-dfa865d9c6f8","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2377976Z"},{"DateTime":"2021-04-16T06:30:00Z","Event":" Producer and Import Prices (MoM)","Country":"CHE","Impact":"Medium","Prev":"0.0%","Consensus":"0.1%","Actual":"","Currency":"CHF","PartitionKey":"mn","RowKey":"b175db34-6250-45cd-9831-37ef2d6382e2","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2387947Z"},{"DateTime":"2021-04-12T12:00:00Z","Event":" Manufacturing Output","Country":"IND","Impact":"Low","Prev":"-2.0%","Consensus":"","Actual":"","Currency":"INR","PartitionKey":"mn","RowKey":"b4545fbb-6773-44bf-8656-1eacab32da02","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2393017Z"},{"DateTime":"2021-04-13T23:00:00Z","Event":" Unemployment Rate","Country":"KOR","Impact":"Low","Prev":"4.0%","Consensus":"","Actual":"","Currency":"KRW","PartitionKey":"mn","RowKey":"b4a5f69e-d7eb-49cd-b1f1-2a7e6d9f57ee","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2369411Z"},{"DateTime":"2021-04-11T23:50:00Z","Event":" Producer Price Index (MoM)","Country":"JPN","Impact":"Low","Prev":"0.6%","Consensus":"0.4%","Actual":"0.8%","Currency":"JPY","PartitionKey":"mn","RowKey":"b676b6c8-7ff1-4a9f-9559-e6447e726efc","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2387639Z"},{"DateTime":"2021-04-16T12:00:00Z","Event":" Net Inflation","Country":"POL","Impact":"Low","Prev":"3.9%","Consensus":"","Actual":"","Currency":"PLN","PartitionKey":"mn","RowKey":"b8fd3996-0fc8-492c-9969-9bb6174991c8","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2389132Z"},{"DateTime":"2021-04-14T09:00:00Z","Event":" Industrial Production w.d.a. (YoY)","Country":"EUR","Impact":"Low","Prev":"0.1%","Consensus":"-0.3%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"b935c49b-f445-495a-9eff-048473f5c2ff","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2374106Z"},{"DateTime":"2021-04-16T06:30:00Z","Event":" Producer and Import Prices (YoY)","Country":"CHE","Impact":"Medium","Prev":"-1.1%","Consensus":"-0.1%","Actual":"","Currency":"CHF","PartitionKey":"mn","RowKey":"b9fed711-2b29-4af3-9f9f-ae896b71608f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2370288Z"},{"DateTime":"2021-04-12T22:45:00Z","Event":" Electronic Card Retail Sales (MoM)","Country":"NZL","Impact":"Low","Prev":"-2.5%","Consensus":"","Actual":"","Currency":"NZD","PartitionKey":"mn","RowKey":"ba2cd71f-91d9-4449-b0a1-e8c64fd37771","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2388835Z"},{"DateTime":"2021-04-14T12:30:00Z","Event":" Export Price Index (MoM)","Country":"USA","Impact":"Low","Prev":"1.6%","Consensus":"1.0%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"bb56fd15-d6c2-4fcd-a3e1-e11262b2c680","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2405852Z"},{"DateTime":"2021-04-16T02:00:00Z","Event":" Fixed Asset Investment (YTD) (YoY)","Country":"CHN","Impact":"Low","Prev":"35.0%","Consensus":"25.3%","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"bb6de717-ccbe-44ae-8cc3-698a35c8f4d4","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2368879Z"},{"DateTime":"2021-04-11T00:00:00Z","Event":" IMF Meeting","Country":"USA","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"bd1f9861-6074-4177-8d97-d28241660369","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2399769Z"},{"DateTime":"2021-04-16T12:30:00Z","Event":" Building Permits (MoM)","Country":"USA","Impact":"High","Prev":"1.72M","Consensus":"1.75M","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"bdaa9f0e-07ef-4da7-8306-b04850879a51","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2405080Z"},{"DateTime":"2021-04-12T07:00:00Z","Event":" New Loans","Country":"CHN","Impact":"Low","Prev":"1360.00B","Consensus":"2450.0B","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"be68cc05-7d87-4cf9-9331-1a7f1a117321","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2389902Z"},{"DateTime":"2021-04-16T07:00:00Z","Event":" Unemployment Rate","Country":"SVK","Impact":"Low","Prev":"7.9%","Consensus":"7.94%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"bee0bffd-df11-488d-a678-c2b5419c0704","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2382463Z"},{"DateTime":"2021-04-16T12:30:00Z","Event":" Housing Starts (MoM)","Country":"USA","Impact":"Medium","Prev":"1.42M","Consensus":"1.613M","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"bf67ea99-5480-46a4-8f3c-c14289bc2be1","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2400121Z"},{"DateTime":"2021-04-16T09:00:00Z","Event":" Consumer Price Index - Core (MoM)","Country":"EUR","Impact":"Medium","Prev":"1.0%","Consensus":"1.0%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"c0073807-a72e-4ade-9ba5-25016488876a","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2395815Z"},{"DateTime":"2021-04-12T22:00:00Z","Event":" NZIER Business Confidence (QoQ)","Country":"NZL","Impact":"Medium","Prev":"-6.0%","Consensus":"","Actual":"","Currency":"NZD","PartitionKey":"mn","RowKey":"c07b4965-ebda-4aba-9775-8114f2773492","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2402066Z"},{"DateTime":"2021-04-12T07:00:00Z","Event":" Current Account Balance","Country":"TUR","Impact":"Low","Prev":"-$1.87B","Consensus":"$-2.575B","Actual":"-$2.61B","Currency":"TRY","PartitionKey":"mn","RowKey":"c16c6ec4-46c9-455f-9ed6-2c9dc576c3c3","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2391983Z"},{"DateTime":"2021-04-13T02:00:00Z","Event":" Trade Balance CNY","Country":"CHN","Impact":"Low","Prev":"247.28B","Consensus":"","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"c1f269de-0ec0-4bcb-95af-96743ed8dbbb","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2408772Z"},{"DateTime":"2021-04-12T23:50:00Z","Event":" Money Supply M2+CD (YoY)","Country":"JPN","Impact":"Low","Prev":"9.6%","Consensus":"9.6%","Actual":"","Currency":"JPY","PartitionKey":"mn","RowKey":"c26db058-3ac1-4486-a779-d149089fb29f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2387473Z"},{"DateTime":"2021-04-16T07:00:00Z","Event":" EcoFin Meeting","Country":"EUR","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"c27feea2-ff1e-4c18-bdf0-0ec31d555528","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2409291Z"},{"DateTime":"2021-04-14T00:30:00Z","Event":" Westpac Consumer Confidence","Country":"AUS","Impact":"Medium","Prev":"2.6%","Consensus":"","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"c7951c1f-48ff-46a2-8659-a23a8a8d1e38","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2378140Z"},{"DateTime":"2021-04-13T12:55:00Z","Event":" Redbook Index (MoM)","Country":"USA","Impact":"Low","Prev":"-17.2%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"caa4c42a-9904-4b5d-87e2-6305601876f8","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2375190Z"},{"DateTime":"2021-04-15T04:00:00Z","Event":" Exports","Country":"IDN","Impact":"Low","Prev":"8.56%","Consensus":"","Actual":"","Currency":"IDR","PartitionKey":"mn","RowKey":"cb05ec7a-290f-4118-be51-e5eafb47ffa0","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2388974Z"},{"DateTime":"2021-04-13T23:50:00Z","Event":" Machinery Orders (MoM)","Country":"JPN","Impact":"Low","Prev":"-4.5%","Consensus":"2.8%","Actual":"","Currency":"JPY","PartitionKey":"mn","RowKey":"cc6dc6a0-f9fa-4f66-bf30-fa2be448857f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2368669Z"},{"DateTime":"2021-04-15T08:00:00Z","Event":" Consumer Price Index (MoM)","Country":"ITA","Impact":"Low","Prev":"0.3%","Consensus":"0.1%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"ccd1f4eb-29bd-4ff7-96aa-fabcf98daf98","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2388108Z"},{"DateTime":"2021-04-16T09:00:00Z","Event":" Consumer Price Index - Core (YoY)","Country":"EUR","Impact":"Medium","Prev":"0.9%","Consensus":"0.9%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"cd30da6a-8236-4034-aab2-25e3109db6d0","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2397491Z"},{"DateTime":"2021-04-14T07:00:00Z","Event":" Core Inflation (MoM)","Country":"SVK","Impact":"Low","Prev":"0.3%","Consensus":"0.5%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"cd87c5cd-d8a6-4cb7-9d6c-6db19d70564d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2409823Z"},{"DateTime":"2021-04-13T08:40:00Z","Event":" 12-Month Letras Auction","Country":"ESP","Impact":"Low","Prev":"-0.506%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"d012c82a-ffcb-4eea-b498-7e37c1d54099","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2379549Z"},{"DateTime":"2021-04-16T16:00:00Z","Event":" Producer Price Index (MoM)","Country":"RUS","Impact":"Low","Prev":"3.5%","Consensus":"","Actual":"","Currency":"RUB","PartitionKey":"mn","RowKey":"d07b14da-70c0-48b9-aad1-5072bd80b903","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2404151Z"},{"DateTime":"2021-04-16T02:00:00Z","Event":" NBS Press Conference","Country":"CHN","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"d459d8b9-7aa9-4444-ba4f-ad5ffd3510ea","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2398895Z"},{"DateTime":"2021-04-13T12:00:00Z","Event":" Current Account","Country":"POL","Impact":"Low","Prev":"€3258.00M","Consensus":"","Actual":"","Currency":"PLN","PartitionKey":"mn","RowKey":"d749c036-4822-4176-93c7-e93462527f22","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2402429Z"},{"DateTime":"2021-04-14T18:00:00Z","Event":" Fed's Beige Book","Country":"USA","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"d903dfa6-558f-43d2-bc57-bc8dcc064637","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2370101Z"},{"DateTime":"2021-04-15T20:00:00Z","Event":" Net Long-Term TIC Flows","Country":"USA","Impact":"Low","Prev":"$90.80B","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"d90989e9-4b99-4632-b377-0d7597e6fc92","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2389742Z"},{"DateTime":"2021-04-15T12:30:00Z","Event":" ADP Employment Change","Country":"CAN","Impact":"Low","Prev":"-100.80K","Consensus":"","Actual":"","Currency":"CAD","PartitionKey":"mn","RowKey":"db7cf784-215c-46bf-852c-3626df9f398f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2367691Z"},{"DateTime":"2021-04-14T19:45:00Z","Event":" Fed's Clarida speech","Country":"USA","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"dca5f4d7-d0c7-4692-9478-fb69cf5a8d05","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2388522Z"},{"DateTime":"2021-04-15T14:00:00Z","Event":" SNB's Maechler speech","Country":"CHE","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"CHF","PartitionKey":"mn","RowKey":"dd6e3182-f4d7-452e-a245-dfc86294efde","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2375967Z"},{"DateTime":"2021-04-13T10:00:00Z","Event":" NFIB Business Optimism Index","Country":"USA","Impact":"Low","Prev":"95.80","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"dd90a9aa-8384-43de-9eac-e6cd9be9f79b","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2390650Z"},{"DateTime":"2021-04-12T09:00:00Z","Event":" Retail Sales (MoM)","Country":"EUR","Impact":"Medium","Prev":"-5.9%","Consensus":"1.5%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"de73a990-1117-4153-bd90-205687e39d55","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2401002Z"},{"DateTime":"2021-04-13T02:00:00Z","Event":" Exports (YoY)","Country":"CHN","Impact":"Medium","Prev":"60.6%","Consensus":"32.7%","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"de8c4e8e-3326-48e0-81af-b94e3ecc635d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2401357Z"},{"DateTime":"2021-04-14T07:00:00Z","Event":" HICP (MoM)","Country":"ESP","Impact":"Low","Prev":"1.9%","Consensus":"-0.6%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"df305d71-712a-41d6-b5c1-0d520306a730","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2380339Z"},{"DateTime":"2021-04-12T06:00:00Z","Event":" Consumer Price Index (YoY)","Country":"DNK","Impact":"Low","Prev":"0.6%","Consensus":"","Actual":"1.0%","Currency":"DKK","PartitionKey":"mn","RowKey":"dfaacfb0-f8dd-4f6f-9d2d-131131090c32","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2380801Z"},{"DateTime":"2021-04-15T08:00:00Z","Event":" Budget Balance","Country":"TUR","Impact":"Low","Prev":"23.17B","Consensus":"","Actual":"","Currency":"TRY","PartitionKey":"mn","RowKey":"e08daaa4-768d-4424-ae0a-14531eb45883","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2396488Z"},{"DateTime":"2021-04-16T12:30:00Z","Event":" Canadian Portfolio Investment in Foreign Securities","Country":"CAN","Impact":"Low","Prev":"$3.54B","Consensus":"","Actual":"","Currency":"CAD","PartitionKey":"mn","RowKey":"e0a534d5-23f1-4e53-9860-c3c07230d559","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2390490Z"},{"DateTime":"2021-04-16T07:00:00Z","Event":" EU Norm Inflation (MoM)","Country":"SVK","Impact":"Low","Prev":"0.3%","Consensus":"0.5%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"e12f52aa-487e-470a-99a8-8b04f22a0696","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2389428Z"},{"DateTime":"2021-04-15T04:00:00Z","Event":" Trade Balance","Country":"IDN","Impact":"Low","Prev":"$2.01B","Consensus":"","Actual":"","Currency":"IDR","PartitionKey":"mn","RowKey":"e19da75e-d469-475c-9ae0-a2f07d8f17ff","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2395636Z"},{"DateTime":"2021-04-16T19:30:00Z","Event":" CFTC GBP NC Net Positions","Country":"GBR","Impact":"Low","Prev":"£20.00K","Consensus":"","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"e19e3da9-b3e2-4c70-8362-929378c80a9f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2385374Z"},{"DateTime":"2021-04-13T12:30:00Z","Event":" Consumer Price Index Core s.a","Country":"USA","Impact":"Medium","Prev":"270.30","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"e1f78c6d-7129-4473-933b-14a5d73f10bf","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2382786Z"},{"DateTime":"2021-04-16T12:30:00Z","Event":" Housing Starts Change","Country":"USA","Impact":"Low","Prev":"-10.3%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"e22eed58-5cfd-4563-a581-2e0934583a0c","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2394818Z"},{"DateTime":"2021-04-14T12:00:00Z","Event":" Budget Fulfilment","Country":"RUS","Impact":"Low","Prev":"-3.8%","Consensus":"","Actual":"","Currency":"RUB","PartitionKey":"mn","RowKey":"e3d25f4c-640a-4807-bc64-8f67946ff8f8","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2403713Z"},{"DateTime":"2021-04-15T08:00:00Z","Event":" Consumer Price Index (YoY)","Country":"POL","Impact":"Low","Prev":"2.4%","Consensus":"","Actual":"","Currency":"PLN","PartitionKey":"mn","RowKey":"e406a5b3-3748-4d65-89a1-f2a430d5d723","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2400291Z"},{"DateTime":"2021-04-13T09:00:00Z","Event":" ZEW Survey – Economic Sentiment","Country":"EUR","Impact":"Medium","Prev":"74.00","Consensus":"73.2","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"e42c1464-f27e-42f3-a2aa-b90b3b4e2e7a","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2410590Z"},{"DateTime":"2021-04-15T06:00:00Z","Event":" Consumer Price Index (MoM)","Country":"DEU","Impact":"Low","Prev":"0.5%","Consensus":"0.5%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"e451c04d-f2ba-41ee-88de-17d4ef17071b","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2377662Z"},{"DateTime":"2021-04-13T21:00:00Z","Event":" Import Price Growth (YoY)","Country":"KOR","Impact":"Low","Prev":"-0.8%","Consensus":"","Actual":"","Currency":"KRW","PartitionKey":"mn","RowKey":"e47a2292-c1e1-4642-9498-ec69c91fed9c","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2396996Z"},{"DateTime":"2021-04-16T12:00:00Z","Event":" Unemployment Rate","Country":"BRA","Impact":"Low","Prev":"14.2%","Consensus":"","Actual":"","Currency":"BRL","PartitionKey":"mn","RowKey":"e81644ea-658f-4290-aa07-b85c56aa55a9","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2382136Z"},{"DateTime":"2021-04-12T06:00:00Z","Event":" Industrial Production (MoM)","Country":"DNK","Impact":"Low","Prev":"1.1%","Consensus":"","Actual":"-1.0%","Currency":"DKK","PartitionKey":"mn","RowKey":"e8289f76-4c17-449d-b55b-607b666c0a5a","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2413650Z"},{"DateTime":"2021-04-15T01:00:00Z","Event":" BoK Interest Rate Decision","Country":"KOR","Impact":"Low","Prev":"0.5%","Consensus":"0.5%","Actual":"","Currency":"KRW","PartitionKey":"mn","RowKey":"e95f9aed-7dc1-4cca-a6d4-e096387330f2","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2379401Z"},{"DateTime":"2021-04-13T07:00:00Z","Event":" Industrial Production (YoY)","Country":"TUR","Impact":"Low","Prev":"11.4%","Consensus":"","Actual":"","Currency":"TRY","PartitionKey":"mn","RowKey":"e96901d3-2168-4f8b-8477-663e1e9eb165","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2410761Z"},{"DateTime":"2021-04-15T08:00:00Z","Event":" Consumer Price Index (MoM)","Country":"POL","Impact":"Low","Prev":"0.5%","Consensus":"","Actual":"","Currency":"PLN","PartitionKey":"mn","RowKey":"e9791e6d-0ad3-42a7-b93a-cdb4aa58fc31","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2407735Z"},{"DateTime":"2021-04-13T01:30:00Z","Event":" National Australia Bank's Business Conditions","Country":"AUS","Impact":"Low","Prev":"15.00","Consensus":"","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"ea31c8c8-50be-4c44-a0e9-7736230ed4a9","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2406645Z"},{"DateTime":"2021-04-14T08:00:00Z","Event":" Industrial Output (YoY)","Country":"HUN","Impact":"Low","Prev":"1.9%","Consensus":"","Actual":"","Currency":"HUF","PartitionKey":"mn","RowKey":"eb109a54-95a1-41d8-96a5-894732e4d7c3","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2373221Z"},{"DateTime":"2021-04-13T10:00:00Z","Event":" Consumer Price Index (MoM)","Country":"PRT","Impact":"Low","Prev":"1.4%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"eb90a0ac-a80a-466e-9c2d-96f820dbcf04","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2375491Z"},{"DateTime":"2021-04-15T08:00:00Z","Event":" Consumer Price Index (EU Norm) (YoY)","Country":"ITA","Impact":"Low","Prev":"0.6%","Consensus":"0.6%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"ed15d8ca-4db3-437f-a4f8-ac31465ed414","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2413856Z"},{"DateTime":"2021-04-14T12:30:00Z","Event":" Import Price Index (YoY)","Country":"USA","Impact":"Medium","Prev":"3.0%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"ed451132-0160-47f7-a980-5b10c7853319","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2371371Z"},{"DateTime":"2021-04-15T01:30:00Z","Event":" Participation Rate","Country":"AUS","Impact":"Low","Prev":"66.1%","Consensus":"66.1%","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"ed7e53ba-9c11-4497-b683-6fd22b4803bd","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2397982Z"},{"DateTime":"2021-04-13T02:00:00Z","Event":" Imports (YoY) CNY","Country":"CHN","Impact":"Low","Prev":"10.3%","Consensus":"","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"edcc772b-3876-44b4-9057-44cda79bf2ce","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2409125Z"},{"DateTime":"2021-04-15T12:30:00Z","Event":" Initial Jobless Claims 4-week average","Country":"USA","Impact":"High","Prev":"723.75K","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"ee6c07f8-243d-467e-aeff-8619f28b913d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2378912Z"},{"DateTime":"2021-04-13T08:00:00Z","Event":" NIESR GDP Estimate (3M)","Country":"GBR","Impact":"Medium","Prev":"-2.0%","Consensus":"","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"eeb29940-b580-4ae3-8388-676c334068ba","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2405390Z"},{"DateTime":"2021-04-14T14:30:00Z","Event":" EIA Crude Oil Stocks Change","Country":"USA","Impact":"Medium","Prev":"-3.52M","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"f07e4304-5726-43d4-87e7-4b32bbead843","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2394295Z"},{"DateTime":"2021-04-12T07:00:00Z","Event":" FDI - Foreign Direct Investment (YTD) (YoY)","Country":"CHN","Impact":"Medium","Prev":"31.5%","Consensus":"38.0%","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"f0e0ad44-e907-4b09-9543-e712d52435c1","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2383133Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Gross Domestic Product (MoM)","Country":"GBR","Impact":"Medium","Prev":"-2.9%","Consensus":"0.6%","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"f1a3be4d-ad7e-418d-bc7b-ea9555bec823","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2391645Z"},{"DateTime":"2021-04-15T06:00:00Z","Event":" Harmonized Index of Consumer Prices (MoM)","Country":"DEU","Impact":"Low","Prev":"0.5%","Consensus":"0.5%","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"f23974cb-6fce-49c1-9143-9c1b1fc1248b","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2374871Z"},{"DateTime":"2021-04-15T01:30:00Z","Event":" Unemployment Rate s.a.","Country":"AUS","Impact":"High","Prev":"5.8%","Consensus":"5.7%","Actual":"","Currency":"AUD","PartitionKey":"mn","RowKey":"f46e71ab-42ab-4e8c-b727-522885d41e49","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2410184Z"},{"DateTime":"2021-04-12T07:45:00Z","Event":" ECB's Panetta speech","Country":"EUR","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"f545bd87-2e5d-467d-a34e-ae972a2e7974","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2378431Z"},{"DateTime":"2021-04-16T14:00:00Z","Event":" Michigan Consumer Sentiment Index","Country":"USA","Impact":"High","Prev":"84.90","Consensus":"88.9","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"f55b8182-6a8a-47c0-a267-409569a4ba1e","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2403387Z"},{"DateTime":"2021-04-13T06:00:00Z","Event":" Industrial Production (YoY)","Country":"GBR","Impact":"Low","Prev":"-4.9%","Consensus":"-4.4%","Actual":"","Currency":"GBP","PartitionKey":"mn","RowKey":"f650c12a-d8fa-4ad8-ba80-cab81b6e3f9e","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2403875Z"},{"DateTime":"2021-04-12T17:00:00Z","Event":" Fed's Rosengren speech","Country":"USA","Impact":"Medium","Prev":"","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"f735c480-30bc-4fbc-aad0-28b9c35fd064","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2376115Z"},{"DateTime":"2021-04-13T09:00:00Z","Event":" ZEW Survey – Economic Sentiment","Country":"DEU","Impact":"Medium","Prev":"76.60","Consensus":"79.5","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"fb26a4db-61dd-46df-a2f3-2ae882c6040b","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2404931Z"},{"DateTime":"2021-04-14T10:40:00Z","Event":" 30-y Bond Auction","Country":"DEU","Impact":"Low","Prev":"0.21%","Consensus":"","Actual":"","Currency":"EUR","PartitionKey":"mn","RowKey":"fc0f90d0-8ec8-4ea7-986f-400f3d03ea83","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2383350Z"},{"DateTime":"2021-04-13T12:30:00Z","Event":" Consumer Price Index ex Food & Energy (YoY)","Country":"USA","Impact":"High","Prev":"1.3%","Consensus":"1.6%","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"fd0e71b8-6936-4711-be54-5fba30cca854","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2402262Z"},{"DateTime":"2021-04-16T02:00:00Z","Event":" Gross Domestic Product (YoY)","Country":"CHN","Impact":"High","Prev":"6.5%","Consensus":"18.8%","Actual":"","Currency":"CNY","PartitionKey":"mn","RowKey":"fd434475-ba40-45e0-a206-5ca6fcb7ea2d","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2369940Z"},{"DateTime":"2021-04-12T06:00:00Z","Event":" Machine Tool Orders (YoY)","Country":"JPN","Impact":"Low","Prev":"36.7%","Consensus":"","Actual":"65.0%","Currency":"JPY","PartitionKey":"mn","RowKey":"fdb89b5a-ec4f-42df-8c75-0f911665177f","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2411118Z"},{"DateTime":"2021-04-15T11:30:00Z","Event":" Trade Deficit Government","Country":"IND","Impact":"Low","Prev":"$14.11B","Consensus":"$14.11B","Actual":"","Currency":"INR","PartitionKey":"mn","RowKey":"ff08b606-b69a-42aa-8fcd-d9ab5e697298","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2376893Z"},{"DateTime":"2021-04-14T11:00:00Z","Event":" MBA Mortgage Applications","Country":"USA","Impact":"Low","Prev":"-5.1%","Consensus":"","Actual":"","Currency":"USD","PartitionKey":"mn","RowKey":"ff7d06e0-9b09-46c5-9d9a-beff260c8073","Timestamp":null,"TimeStamp":"2021-04-12T07:38:24.2380635Z"}] \ No newline at end of file